diff --git a/CHANGELOG.md b/CHANGELOG.md index 7eb0b3d8..f8c9fcb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- changed: Add index for orderId +- changed: Add EVM chainId, pluginId, and tokenId fields to StandardTx +- changed: Update Lifi to provide chainId, pluginId, and tokenId +- changed: Use rates V3 for transactions with pluginId/tokenId +- fixed: Moonpay by adding Revolut payment type +- fixed: Use v2 rates API + +## 0.2.0 + - added: Add Lifi reporting - added: Added `/v1/getTxInfo` route. - added: Paybis support diff --git a/package.json b/package.json index e91f7c0b..7791671b 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,10 @@ "prepare": "./scripts/prepare.sh && npm-run-all clean configure -p build.* && npm run setup", "setup": "node -r sucrase/register src/initDbs.ts", "start": "node -r sucrase/register src/indexQuery.ts", - "start:cache": "node -r sucrase/register src/indexCache.ts", - "start:rates": "node -r sucrase/register src/indexRates.ts", - "start:api": "node -r sucrase/register src/indexApi.ts", - "start:destroyPartition": "node -r sucrase/register src/bin/destroyPartition.ts", + "start.cache": "node -r sucrase/register src/indexCache.ts", + "start.rates": "node -r sucrase/register src/indexRates.ts", + "start.api": "node -r sucrase/register src/indexApi.ts", + "start.destroyPartition": "node -r sucrase/register src/bin/destroyPartition.ts", "stats": "node -r sucrase/register src/bin/partitionStats.ts", "test": "mocha -r sucrase/register 'test/**/*.test.ts'", "demo": "parcel serve src/demo/index.html", @@ -78,6 +78,7 @@ "@typescript-eslint/eslint-plugin": "^5.36.2", "@typescript-eslint/parser": "^5.36.2", "assert": "^2.0.0", + "async-mutex": "^0.5.0", "browserify-zlib": "^0.2.0", "chai": "^4.3.4", "eslint": "^8.19.0", diff --git a/src/bin/destroyPartition.ts b/src/bin/destroyPartition.ts index 7c8d0ad9..7e24b3e5 100644 --- a/src/bin/destroyPartition.ts +++ b/src/bin/destroyPartition.ts @@ -3,7 +3,7 @@ import js from 'jsonfile' import nano from 'nano' import { pagination } from '../dbutils' -import { datelog } from '../util' +import { createScopedLog, datelog } from '../util' const config = js.readFileSync('./config.json') const nanoDb = nano(config.couchDbFullpath) @@ -43,10 +43,11 @@ async function main(partitionName: string): Promise { return } + const log = createScopedLog('destroy', partitionName) try { - await pagination(transactions, reportsTransactions) - datelog(`Successfully Deleted: ${transactions.length} docs`) - datelog(`Successfully Deleted: partition ${partitionName}`) + await pagination(transactions, reportsTransactions, log) + log(`Successfully Deleted: ${transactions.length} docs`) + log(`Successfully Deleted: partition ${partitionName}`) // Delete progress Cache const split = partitionName.split('_') @@ -56,7 +57,9 @@ async function main(partitionName: string): Promise { datelog(`Successfully Deleted: progress cache ${progress._id}`) } catch (e) { datelog(e) + process.exit(1) } + process.exit(0) } main(process.argv[2]).catch(e => datelog(e)) diff --git a/src/bin/fioPromo/fioLookup.ts b/src/bin/fioPromo/fioLookup.ts index 513bd033..2afe68f6 100644 --- a/src/bin/fioPromo/fioLookup.ts +++ b/src/bin/fioPromo/fioLookup.ts @@ -5,6 +5,7 @@ import path from 'path' import { queryChangeNow } from '../../partners/changenow' import { PluginParams, StandardTx } from '../../types' +import { createScopedLog } from '../../util' import { defaultSettings } from './fioInfo' let addressList: string[] = [] @@ -48,12 +49,14 @@ export async function getFioTransactions( dateFrom: Date, dateTo: Date ): Promise { + const log = createScopedLog('fio', 'changenow') // Get public keys from offset const pluginConfig: PluginParams = { settings: { dateFrom, dateTo, to: currencyCode }, apiKeys: { changenowApiKey: config.changenowApiKey - } + }, + log } const txnList = await queryChangeNow(pluginConfig) diff --git a/src/bin/testpartner.ts b/src/bin/testpartner.ts index 3acc83a8..2155dac2 100644 --- a/src/bin/testpartner.ts +++ b/src/bin/testpartner.ts @@ -1,16 +1,28 @@ -import { thorchain as plugin } from '../partners/thorchain' -import { PluginParams } from '../types.js' +import { PluginParams } from '../types' +import { createScopedLog } from '../util' -const pluginParams: PluginParams = { - settings: { - offset: 0 - }, - apiKeys: { - thorchainAddress: '' +async function main(): Promise { + const partnerId = process.argv[2] + if (partnerId == null) { + console.log( + 'Usage: node -r sucrase/register src/bin/testpartner.ts ' + ) + process.exit(1) + } + + const pluginParams: PluginParams = { + settings: {}, + apiKeys: {}, + log: createScopedLog('edge', partnerId) + } + + // Dynamically import the partner plugin + const pluginModule = await import(`../partners/${partnerId}`) + const plugin = pluginModule[partnerId] + if (plugin?.queryFunc == null) { + throw new Error(`Plugin ${partnerId} does not have a queryFunc`) } -} -async function main(): Promise { const result = await plugin.queryFunc(pluginParams) console.log(JSON.stringify(result, null, 2)) } diff --git a/src/dbutils.ts b/src/dbutils.ts index 7f8dbc15..e59f9a66 100644 --- a/src/dbutils.ts +++ b/src/dbutils.ts @@ -1,10 +1,9 @@ import { asArray, asNumber, asObject, asString } from 'cleaners' import nano from 'nano' -import { getAnalytics } from './apiAnalytics' import { config } from './config' -import { AnalyticsResult, asCacheQuery } from './types' -import { datelog, promiseTimeout } from './util' +import { AnalyticsResult, asCacheQuery, ScopedLog } from './types' +import { promiseTimeout } from './util' const BATCH_ADVANCE = 100 const SIX_DAYS_IN_SECONDS = 6 * 24 * 60 * 60 @@ -25,7 +24,8 @@ export type DbReq = ReturnType export const pagination = async ( txArray: any[], - partition: nano.DocumentScope + partition: nano.DocumentScope, + log: ScopedLog ): Promise => { let numErrors = 0 for (let offset = 0; offset < txArray.length; offset += BATCH_ADVANCE) { @@ -37,19 +37,20 @@ export const pagination = async ( 'partition.bulk', partition.bulk({ docs: txArray.slice(offset, offset + advance) - }) + }), + log ) - datelog(`Processed ${offset + advance} txArray.`) + log(`[pagination] Processed ${offset + advance} txArray.`) for (const doc of docs) { if (doc.error != null) { - datelog( - `There was an error in the batch ${doc.error}. id: ${doc.id}. revision: ${doc.rev}` + log.error( + `[pagination] There was an error in the batch ${doc.error}. id: ${doc.id}. revision: ${doc.rev}` ) numErrors++ } } } - datelog(`total errors: ${numErrors}`) + log(`[pagination] total errors: ${numErrors}`) } export const cacheAnalytic = async ( diff --git a/src/demo/demo.tsx b/src/demo/demo.tsx index 34074bbb..4490841f 100644 --- a/src/demo/demo.tsx +++ b/src/demo/demo.tsx @@ -60,7 +60,7 @@ class App extends Component< async componentDidMount(): Promise { Object.assign(document.body.style, body) - if (this.state.apiKey !== '') { + if (this.state.apiKey != null && this.state.apiKey.length > 2) { await this.getAppId() } } diff --git a/src/demo/partners.ts b/src/demo/partners.ts index 90d24146..14e7dc4b 100644 --- a/src/demo/partners.ts +++ b/src/demo/partners.ts @@ -105,6 +105,10 @@ export default { type: 'fiat', color: '#99A5DE' }, + rango: { + type: 'swap', + color: '#5891EE' + }, safello: { type: 'fiat', color: deprecated diff --git a/src/initDbs.ts b/src/initDbs.ts index 15881293..ab3252d9 100644 --- a/src/initDbs.ts +++ b/src/initDbs.ts @@ -39,6 +39,7 @@ function fieldsToDesignDocs( const transactionIndexes: DesignDocumentMap = { ...fieldsToDesignDocs(['isoDate']), + ...fieldsToDesignDocs(['orderId']), ...fieldsToDesignDocs(['status']), ...fieldsToDesignDocs(['status', 'depositCurrency', 'isoDate']), ...fieldsToDesignDocs([ @@ -54,6 +55,10 @@ const transactionIndexes: DesignDocumentMap = { ...fieldsToDesignDocs(['status', 'usdValue', 'timestamp']), ...fieldsToDesignDocs(['usdValue']), ...fieldsToDesignDocs(['timestamp']), + ...fieldsToDesignDocs(['status', 'depositChainPluginId']), + ...fieldsToDesignDocs(['status', 'payoutChainPluginId']), + ...fieldsToDesignDocs(['status', 'depositChainPluginId', 'depositTokenId']), + ...fieldsToDesignDocs(['status', 'payoutChainPluginId', 'payoutTokenId']), ...fieldsToDesignDocs(['depositAddress'], { noPartitionVariant: true }), ...fieldsToDesignDocs(['payoutAddress'], { noPartitionVariant: true }), ...fieldsToDesignDocs(['payoutAddress', 'isoDate'], { @@ -125,20 +130,14 @@ const options: SetupDatabaseOptions = { } export async function initDbs(): Promise { - if (config.couchUris != null) { - const pool = connectCouch(config.couchMainCluster, config.couchUris) - await setupDatabase(pool, couchSettingsSetup, options) - await Promise.all( - databases.map(async setup => await setupDatabase(pool, setup, options)) - ) - } else { - await Promise.all( - databases.map( - async setup => - await setupDatabase(config.couchDbFullpath, setup, options) - ) - ) + if (config.couchUris == null) { + throw new Error('couchUris is not set') } + const pool = connectCouch(config.couchMainCluster, config.couchUris) + await setupDatabase(pool, couchSettingsSetup, options) + await Promise.all( + databases.map(async setup => await setupDatabase(pool, setup, options)) + ) console.log('Done') process.exit(0) } diff --git a/src/partners/banxa.ts b/src/partners/banxa.ts index ca29cdcf..e1de25ea 100644 --- a/src/partners/banxa.ts +++ b/src/partners/banxa.ts @@ -17,10 +17,233 @@ import { PartnerPlugin, PluginParams, PluginResult, + ScopedLog, StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { + ChainNameToPluginIdMapping, + createTokenId, + EdgeTokenId, + tokenTypes +} from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS, REVERSE_EVM_CHAIN_IDS } from '../util/chainIds' + +// Map Banxa blockchain.id (from v2 API) to Edge pluginId +// First we try to use the numeric chain ID from the 'network' field, +// falling back to this mapping for non-EVM chains +const BANXA_BLOCKCHAIN_TO_PLUGIN_ID: ChainNameToPluginIdMapping = { + ALGO: 'algorand', + ARB: 'arbitrum', + AVAX: 'avalanche', + 'AVAX-C': 'avalanche', + BASE: 'base', + BCH: 'bitcoincash', + BOB: 'bobevm', + BSC: 'binancesmartchain', + BTC: 'bitcoin', + CELO: 'celo', + DOGE: 'dogecoin', + DOT: 'polkadot', + ETC: 'ethereumclassic', + ETH: 'ethereum', + FTM: 'fantom', + HBAR: 'hedera', + LN: 'bitcoin', // Lightning Network maps to bitcoin + LTC: 'litecoin', + MATIC: 'polygon', + OP: 'optimism', + OPTIMISM: 'optimism', + POL: 'polygon', + SOL: 'solana', + SUI: 'sui', + TON: 'ton', + TRX: 'tron', + XLM: 'stellar', + XRP: 'ripple', + XTZ: 'tezos', + ZKSYNC: 'zksync', + ZKSYNC2: 'zksync' +} + +// Cleaner for Banxa v2 API blockchain +const asBanxaBlockchain = asObject({ + id: asString, + address: asOptional(asString), + network: asOptional(asString) +}) + +// Cleaner for Banxa v2 API coin +const asBanxaCoin = asObject({ + id: asString, + blockchains: asArray(asBanxaBlockchain) +}) + +// Cleaner for Banxa v2 API response (array of coins) +const asBanxaCryptoResponse = asArray(asBanxaCoin) + +// Cache for Banxa coins data from v2 API +// Key: `${coinId}-${blockchainId}` -> { contractAddress, pluginId } +interface CachedAssetInfo { + contractAddress: string | null + pluginId: string | undefined +} +let banxaCoinsCache: Map | null = null + +// Static fallback for historical coins no longer in the v2 API +const BANXA_HISTORICAL_COINS: Record = { + // MATIC was renamed to POL + 'MATIC-MATIC': { contractAddress: null, pluginId: 'polygon' }, + 'MATIC-ETH': { + contractAddress: '0x7d1afa7b718fb893db30a3abc0cfc608aacfebb0', + pluginId: 'ethereum' + }, + // OMG (OmiseGO) delisted + 'OMG-ETH': { + contractAddress: '0xd26114cd6EE289AccF82350c8d8487fedB8A0C07', + pluginId: 'ethereum' + }, + // RLUSD on XRP Ledger + 'RLUSD-XRP': { + contractAddress: 'rMxCKbEDwqr76QuheSUMdEGf4B9xJ8m5De', + pluginId: 'ripple' + } +} + +/** + * Fetch coins from Banxa v2 API and build cache + */ +async function fetchBanxaCoins( + partnerId: string, + apiKeyV2: string, + log: ScopedLog +): Promise> { + if (banxaCoinsCache != null) { + return banxaCoinsCache + } + + const cache = new Map() + + // Fetch both buy and sell to get all coins + for (const orderType of ['buy', 'sell']) { + const url = `https://api.banxa.com/${partnerId}/v2/crypto/${orderType}` + const response = await retryFetch(url, { + headers: { + 'x-api-key': apiKeyV2, + Accept: 'application/json' + } + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error( + `Failed to fetch Banxa ${orderType} coins: ${response.status} - ${errorText}` + ) + } + + const rawCoins = await response.json() + const coins = asBanxaCryptoResponse(rawCoins) + + for (const coin of coins) { + for (const blockchain of coin.blockchains) { + const key = `${coin.id.toUpperCase()}-${blockchain.id.toUpperCase()}` + + // Skip if already cached + if (cache.has(key)) continue + + // Determine pluginId from network (chain ID) or blockchain id + let pluginId: string | undefined + const networkId = blockchain.network + if (networkId != null) { + // Try to parse as numeric chain ID + const chainIdNum = parseInt(networkId, 10) + if (!isNaN(chainIdNum)) { + pluginId = REVERSE_EVM_CHAIN_IDS[chainIdNum] + } + } + // Fall back to blockchain ID mapping + if (pluginId == null) { + pluginId = BANXA_BLOCKCHAIN_TO_PLUGIN_ID[blockchain.id.toUpperCase()] + } + + if (pluginId == null) { + continue + } + + // Determine contract address + // null, empty, "0x0000...", or non-hex addresses (like bip122:...) mean native gas token + // Also, if coin ID matches blockchain ID (e.g. HBAR-HBAR), it's the native coin + let contractAddress: string | null = null + const isNativeCoin = + coin.id.toUpperCase() === blockchain.id.toUpperCase() + if ( + !isNativeCoin && + blockchain.address != null && + blockchain.address !== '' && + blockchain.address !== '0x0000000000000000000000000000000000000000' && + blockchain.address.startsWith('0x') // Only EVM-style addresses are contracts + ) { + contractAddress = blockchain.address + } + + cache.set(key, { contractAddress, pluginId }) + } + } + } + + banxaCoinsCache = cache + log(`Loaded ${cache.size} coin/blockchain combinations from API`) + return cache +} + +interface EdgeAssetInfo { + chainPluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId +} + +/** + * Get Edge asset info from Banxa blockchain code and coin code + * Uses cached data from v2 API + */ +function getAssetInfo(blockchainCode: string, coinCode: string): EdgeAssetInfo { + const cacheKey = `${coinCode.toUpperCase()}-${blockchainCode.toUpperCase()}` + + // Try API cache first, then historical fallback + let cachedInfo = banxaCoinsCache?.get(cacheKey) + if (cachedInfo == null) { + cachedInfo = BANXA_HISTORICAL_COINS[cacheKey] + } + if (cachedInfo == null) { + throw new Error( + `Unknown Banxa coin/blockchain: ${coinCode} on ${blockchainCode}` + ) + } + + const { contractAddress, pluginId: chainPluginId } = cachedInfo + + if (chainPluginId == null) { + throw new Error(`Unknown Banxa blockchain: ${blockchainCode}`) + } + + // Get evmChainId if this is an EVM chain + const evmChainId = EVM_CHAIN_IDS[chainPluginId] + + // Determine tokenId + let tokenId: EdgeTokenId = null + if (contractAddress != null) { + const tokenType = tokenTypes[chainPluginId] + if (tokenType == null) { + throw new Error( + `Unknown tokenType for chainPluginId ${chainPluginId} (coin: ${coinCode})` + ) + } + tokenId = createTokenId(tokenType, coinCode.toUpperCase(), contractAddress) + } + + return { chainPluginId, evmChainId, tokenId } +} export const asBanxaParams = asObject({ settings: asObject({ @@ -28,6 +251,8 @@ export const asBanxaParams = asObject({ }), apiKeys: asObject({ apiKey: asString, + partnerId: asString, + apiKeyV2: asString, secret: asString, partnerUrl: asString }) @@ -58,7 +283,11 @@ const asBanxaTx = asObject({ coin_code: asString, order_type: asString, payment_type: asString, - wallet_address: asMaybe(asString, '') + wallet_address: asMaybe(asString, ''), + blockchain: asObject({ + code: asString, + description: asString + }) }) const asBanxaResult = asObject({ @@ -68,7 +297,7 @@ const asBanxaResult = asObject({ }) const MAX_ATTEMPTS = 1 -const PAGE_LIMIT = 100 +const PAGE_LIMIT = 200 const ONE_DAY_MS = 1000 * 60 * 60 * 24 const ROLLBACK = ONE_DAY_MS * 7 // 7 days @@ -85,9 +314,10 @@ const statusMap: { [key in BanxaStatus]: Status } = { export async function queryBanxa( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const ssFormatTxs: StandardTx[] = [] const { settings, apiKeys } = asBanxaParams(pluginParams) - const { apiKey, partnerUrl, secret } = apiKeys + const { apiKey, partnerId, partnerUrl, secret } = apiKeys const { latestIsoDate } = settings if (apiKey == null) { @@ -110,8 +340,8 @@ export async function queryBanxa( let attempt = 0 try { while (true) { - datelog( - `BANXA: Querying ${startDate}->${endDate}, limit=${PAGE_LIMIT} page=${page} attempt=${attempt}` + log( + `Querying ${startDate}->${endDate}, limit=${PAGE_LIMIT} page=${page} attempt=${attempt}` ) const response = await fetchBanxaAPI( partnerUrl, @@ -127,13 +357,13 @@ export async function queryBanxa( if (!response.ok) { attempt++ const delay = 2000 * attempt - datelog( - `BANXA: Response code ${response.status}. Retrying after ${delay / + log.warn( + `Response code ${response.status}. Retrying after ${delay / 1000} second snooze...` ) await snooze(delay) if (attempt === MAX_ATTEMPTS) { - datelog(`BANXA: Retry Limit reached for date ${startDate}.`) + log.error(`Retry Limit reached for date ${startDate}.`) const text = await response.text() throw new Error(text) @@ -144,7 +374,7 @@ export async function queryBanxa( const reply = await response.json() const jsonObj = asBanxaResult(reply) const txs = jsonObj.data.orders - processBanxaOrders(txs, ssFormatTxs) + await processBanxaOrders(txs, ssFormatTxs, pluginParams, log) if (txs.length < PAGE_LIMIT) { break } @@ -153,7 +383,7 @@ export async function queryBanxa( const newStartTs = new Date(endDate).getTime() startDate = new Date(newStartTs).toISOString() } catch (e) { - datelog(String(e)) + log.error(String(e)) endDate = startDate // We can safely save our progress since we go from oldest to newest. @@ -204,19 +434,21 @@ async function fetchBanxaAPI( return await retryFetch(`${partnerUrl}${apiQuery}`, { headers: headers }) } -function processBanxaOrders( +async function processBanxaOrders( rawtxs: unknown[], - ssFormatTxs: StandardTx[] -): void { + ssFormatTxs: StandardTx[], + pluginParams: PluginParams, + log: ScopedLog +): Promise { let numComplete = 0 let newestIsoDate = new Date(0).toISOString() let oldestIsoDate = new Date(9999999999999).toISOString() for (const rawTx of rawtxs) { let standardTx: StandardTx try { - standardTx = processBanxaTx(rawTx) + standardTx = await processBanxaTx(rawTx, pluginParams) } catch (e) { - datelog(String(e)) + log.error(String(e)) throw e } @@ -233,8 +465,8 @@ function processBanxaOrders( } } if (rawtxs.length > 1) { - datelog( - `BANXA: Processed ${ + log( + `Processed ${ rawtxs.length }, #complete=${numComplete} oldest=${oldestIsoDate.slice( 0, @@ -242,13 +474,25 @@ function processBanxaOrders( )} newest=${newestIsoDate.slice(0, 16)}` ) } else { - datelog(`BANXA: Processed ${rawtxs.length}`) + log(`Processed ${rawtxs.length}`) } } -export function processBanxaTx(rawTx: unknown): StandardTx { +export async function processBanxaTx( + rawTx: unknown, + pluginParams: PluginParams +): Promise { + const { log } = pluginParams const banxaTx: BanxaTx = asBanxaTx(rawTx) const { isoDate, timestamp } = smartIsoDateFromTimestamp(banxaTx.created_at) + const { apiKeys } = asBanxaParams(pluginParams) + const { apiKeyV2, partnerId } = apiKeys + + // Get apiKeyV2 from pluginParams (banxa3 partner) + // For backfillAssetInfo, this comes from the banxa3 partner config + if (apiKeyV2 == null || partnerId == null) { + throw new Error('Banxa apiKeyV2 required for asset info lookup') + } // Flip the amounts if the order is a SELL let payoutAddress @@ -269,6 +513,28 @@ export function processBanxaTx(rawTx: unknown): StandardTx { const paymentType = getFiatPaymentType(banxaTx) + // Get asset info for the crypto side + // For buy: payout is crypto + // For sell: deposit is crypto + const blockchainCode = banxaTx.blockchain.code + const coinCode = banxaTx.coin_code + + await fetchBanxaCoins(partnerId, apiKeyV2, log) + + const cryptoAssetInfo = getAssetInfo(blockchainCode, coinCode) + + // For buy transactions: deposit is fiat (no crypto info), payout is crypto + // For sell transactions: deposit is crypto, payout is fiat (no crypto info) + const depositAsset = + direction === 'sell' + ? cryptoAssetInfo + : { chainPluginId: undefined, evmChainId: undefined, tokenId: undefined } + + const payoutAsset = + direction === 'buy' + ? cryptoAssetInfo + : { chainPluginId: undefined, evmChainId: undefined, tokenId: undefined } + const standardTx: StandardTx = { status: statusMap[banxaTx.status], orderId: banxaTx.id, @@ -276,6 +542,9 @@ export function processBanxaTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: inputCurrency, + depositChainPluginId: depositAsset.chainPluginId, + depositEvmChainId: depositAsset.evmChainId, + depositTokenId: depositAsset.tokenId, depositAmount: inputAmount, direction, exchangeType: 'fiat', @@ -283,6 +552,9 @@ export function processBanxaTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress, payoutCurrency: outputCurrency, + payoutChainPluginId: payoutAsset.chainPluginId, + payoutEvmChainId: payoutAsset.evmChainId, + payoutTokenId: payoutAsset.tokenId, payoutAmount: outputAmount, timestamp, isoDate, @@ -332,6 +604,17 @@ function getFiatPaymentType(tx: BanxaTx): FiatPaymentType { return 'googlepay' case 'iDEAL Transfer': return 'ideal' + case 'ZeroHash ACH Sell': + case 'Fortress/Plaid ACH': + return 'ach' + case 'Manual Payment (Turkey)': + return 'turkishbank' + case 'ClearJunction Sell Sepa': + return 'sepa' + case 'Dlocal Brazil PIX': + return 'pix' + case 'DLocal South Africa IO': + return 'ozow' default: throw new Error(`Unknown payment method: ${tx.payment_type} for ${tx.id}`) } diff --git a/src/partners/bitaccess.ts b/src/partners/bitaccess.ts index afd62bdd..d01bfc5c 100644 --- a/src/partners/bitaccess.ts +++ b/src/partners/bitaccess.ts @@ -11,7 +11,6 @@ import crypto from 'crypto' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' import { queryDummy } from './dummy' const asBitaccessTx = asObject({ @@ -42,6 +41,7 @@ const QUERY_LOOKBACK = 60 * 60 * 24 * 5 // 5 days export async function queryBitaccess( pluginParams: PluginParams ): Promise { + const { log } = pluginParams let lastTimestamp = 0 if (typeof pluginParams.settings.lastTimestamp === 'number') { lastTimestamp = pluginParams.settings.lastTimestamp @@ -104,7 +104,7 @@ export async function queryBitaccess( } page++ } catch (e) { - datelog(e) + log.error(String(e)) throw e } } @@ -145,6 +145,9 @@ export function processBitaccessTx(rawTx: unknown): StandardTx { depositTxid, depositAddress: tx.deposit_address, depositCurrency: tx.deposit_currency.toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.deposit_amount, direction: tx.trade_type, exchangeType: 'fiat', @@ -152,6 +155,9 @@ export function processBitaccessTx(rawTx: unknown): StandardTx { payoutTxid, payoutAddress: tx.withdrawal_address, payoutCurrency: tx.withdrawal_currency.toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.withdrawal_amount, timestamp, isoDate: tx.updated_at, diff --git a/src/partners/bitrefill.ts b/src/partners/bitrefill.ts index 850dbd28..0d73bc02 100644 --- a/src/partners/bitrefill.ts +++ b/src/partners/bitrefill.ts @@ -1,4 +1,3 @@ -import { div } from 'biggystring' import { asArray, asBoolean, @@ -6,52 +5,176 @@ import { asObject, asOptional, asString, - asUnknown + asUnknown, + asValue } from 'cleaners' import fetch from 'node-fetch' -import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, safeParseFloat } from '../util' +import { + PartnerPlugin, + PluginParams, + PluginResult, + StandardTx, + Status +} from '../types' +import { smartIsoDateFromTimestamp } from '../util' +import { EdgeTokenId } from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +const asBitrefillStatus = asValue( + 'unpaid', + 'delivered', + 'sealed', + 'created', + 'permanent_failure', + 'refunded' +) const asBitrefillTx = asObject({ + status: asBitrefillStatus, paymentReceived: asBoolean, expired: asBoolean, sent: asBoolean, invoiceTime: asNumber, - satoshiPrice: asOptional(asNumber), + btcPrice: asString, + altcoinPrice: asOptional(asString), value: asString, currency: asString, country: asString, coinCurrency: asString, - receivedPaymentAltcoin: asOptional(asNumber), + paymentMethod: asString, + // receivedPaymentAltcoin: asOptional(asNumber), orderId: asString, usdPrice: asNumber }) -// Partial type for Bitrefill txs for pre-processing -const asPreBitrefillTx = asObject({ - expired: asBoolean, - paymentReceived: asBoolean, - sent: asBoolean, - status: asString -}) +type BitrefillTx = ReturnType +type BitrefillStatus = ReturnType +const statusMap: { [key in BitrefillStatus]: Status } = { + unpaid: 'pending', + created: 'pending', + delivered: 'complete', + refunded: 'refunded', + permanent_failure: 'failed', + sealed: 'complete' +} const asBitrefillResult = asObject({ nextUrl: asOptional(asString), orders: asArray(asUnknown) }) -const multipliers: { [key: string]: string } = { - BTC: '100000000', - ETH: '1000000', - LTC: '100000000', - DASH: '100000000', - DOGE: '100000000' +const countryCodeMap: { [key: string]: string | null } = { + argentina: 'AR', + australia: 'AU', + austria: 'AT', + bangladesh: 'BD', + belgium: 'BE', + bolivia: 'BO', + brazil: 'BR', + canada: 'CA', + chile: 'CL', + china: 'CN', + colombia: 'CO', + 'costa-rica': 'CR', + 'czech-republic': 'CZ', + denmark: 'DK', + 'dominican-republic': 'DO', + ecuador: 'EC', + egypt: 'EG', + 'el-salvador': 'SV', + finland: 'FI', + france: 'FR', + germany: 'DE', + ghana: 'GH', + greece: 'GR', + guatemala: 'GT', + honduras: 'HN', + 'hong-kong': 'HK', + hungary: 'HU', + india: 'IN', + indonesia: 'ID', + ireland: 'IE', + israel: 'IL', + italy: 'IT', + japan: 'JP', + kenya: 'KE', + malaysia: 'MY', + mexico: 'MX', + morocco: 'MA', + netherlands: 'NL', + 'new-zealand': 'NZ', + nicaragua: 'NI', + nigeria: 'NG', + norway: 'NO', + pakistan: 'PK', + panama: 'PA', + paraguay: 'PY', + peru: 'PE', + philippines: 'PH', + poland: 'PL', + portugal: 'PT', + romania: 'RO', + russia: 'RU', + 'saudi-arabia': 'SA', + singapore: 'SG', + 'south-africa': 'ZA', + 'south-korea': 'KR', + spain: 'ES', + sweden: 'SE', + switzerland: 'CH', + taiwan: 'TW', + thailand: 'TH', + turkey: 'TR', + ukraine: 'UA', + 'united-arab-emirates': 'AE', + 'united-kingdom': 'GB', + uruguay: 'UY', + usa: 'US', + venezuela: 'VE', + vietnam: 'VN', + eu: 'EU', + international: null +} + +const paymentMethodMap: { + [key: string]: { + pluginId: string + tokenId: EdgeTokenId + currencyCode: string + } +} = { + bitcoin: { pluginId: 'bitcoin', tokenId: null, currencyCode: 'BTC' }, + dash: { pluginId: 'dash', tokenId: null, currencyCode: 'DASH' }, + ethereum: { pluginId: 'ethereum', tokenId: null, currencyCode: 'ETH' }, + litecoin: { pluginId: 'litecoin', tokenId: null, currencyCode: 'LTC' }, + dogecoin: { pluginId: 'dogecoin', tokenId: null, currencyCode: 'DOGE' }, + usdc_erc20: { + pluginId: 'ethereum', + tokenId: 'a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + currencyCode: 'USDC' + }, + usdc_polygon: { + pluginId: 'polygon', + tokenId: '3c499c542cef5e3811e1192ce70d8cc03d5c3359', + currencyCode: 'USDC' + }, + usdt_erc20: { + pluginId: 'ethereum', + tokenId: 'dac17f958d2ee523a2206206994597c13d831ec7', + currencyCode: 'USDT' + }, + usdt_trc20: { + pluginId: 'tron', + tokenId: 'TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t', + currencyCode: 'USDT' + } } export async function queryBitrefill( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const MAX_ITERATIONS = 20 let username = '' let password = '' @@ -73,7 +196,7 @@ export async function queryBitrefill( } const standardTxs: StandardTx[] = [] - let url = `https://api.bitrefill.com/v1/orders/` + let url = `https://api.bitrefill.com/v1/orders/?limit=150` let count = 0 while (true) { let jsonObj: ReturnType @@ -82,22 +205,16 @@ export async function queryBitrefill( method: 'GET', headers }) - jsonObj = asBitrefillResult(await result.json()) + const json = await result.json() + jsonObj = asBitrefillResult(json) } catch (e) { - datelog(e) + log.error(String(e)) break } const txs = jsonObj.orders for (const rawTx of txs) { - // Pre-process the tx to see if it's meets criteria for inclusion: - const preTx = asPreBitrefillTx(rawTx) - if (preTx.status === 'unpaid') { - continue - } - if (preTx.paymentReceived && !preTx.expired && preTx.sent) { - const standardTx = processBitrefillTx(rawTx) - standardTxs.push(standardTx) - } + const standardTx = processBitrefillTx(rawTx, pluginParams) + standardTxs.push(standardTx) } if (count > MAX_ITERATIONS) { @@ -126,31 +243,62 @@ export const bitrefill: PartnerPlugin = { pluginId: 'bitrefill' } -export function processBitrefillTx(rawTx: unknown): StandardTx { - const tx = asBitrefillTx(rawTx) - const timestamp = tx.invoiceTime / 1000 +export function processBitrefillTx( + rawTx: unknown, + pluginParams: PluginParams +): StandardTx { + const { log } = pluginParams + let tx: BitrefillTx + try { + tx = asBitrefillTx(rawTx) + } catch (e) { + throw new Error(`${String(e)}: ${JSON.stringify(rawTx)}`) + } + const { isoDate } = smartIsoDateFromTimestamp(tx.invoiceTime) + const countryCode = countryCodeMap[tx.country] + + if (tx.altcoinPrice != null) { + log( + `${tx.orderId}: ${isoDate} ${countryCode} ${tx.status} ${tx.paymentMethod} alt:${tx.altcoinPrice}` + ) + } else { + log( + `${tx.orderId}: ${isoDate} ${countryCode} ${tx.status} ${tx.paymentMethod} btc:${tx.btcPrice}` + ) + } + const edgeAsset = paymentMethodMap[tx.paymentMethod] - const inputCurrency: string = tx.coinCurrency.toUpperCase() - if (typeof multipliers[inputCurrency] !== 'string') { - throw new Error(inputCurrency + ' has no multipliers') + if (edgeAsset == null) { + throw new Error(`${tx.orderId}: ${tx.paymentMethod} has no payment method`) + } + if (countryCode === undefined) { + throw new Error(`${tx.orderId}: ${tx.country} has no country code`) } - let depositAmountStr = tx.satoshiPrice?.toString() - if (typeof inputCurrency === 'string' && inputCurrency !== 'BTC') { - depositAmountStr = tx.receivedPaymentAltcoin?.toString() + const evmChainId = EVM_CHAIN_IDS[edgeAsset.pluginId] + + const timestamp = tx.invoiceTime / 1000 + + const { paymentMethod } = tx + let depositAmountStr: string | undefined + if (paymentMethod === 'bitcoin') { + depositAmountStr = tx.btcPrice + } else if (tx.altcoinPrice != null) { + depositAmountStr = tx.altcoinPrice } if (depositAmountStr == null) { throw new Error(`Missing depositAmount for tx: ${tx.orderId}`) } - const depositAmount = safeParseFloat( - div(depositAmountStr, multipliers[inputCurrency], 8) - ) + const depositAmount = Number(depositAmountStr) const standardTx: StandardTx = { - status: 'complete', + status: statusMap[tx.status], orderId: tx.orderId, - countryCode: tx.country.toUpperCase(), + countryCode, depositTxid: undefined, depositAddress: undefined, - depositCurrency: inputCurrency, + depositCurrency: edgeAsset.currencyCode, + depositChainPluginId: edgeAsset.pluginId, + depositEvmChainId: evmChainId, + depositTokenId: edgeAsset.tokenId, depositAmount, direction: 'sell', exchangeType: 'fiat', @@ -158,9 +306,12 @@ export function processBitrefillTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: tx.currency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: parseInt(tx.value), timestamp, - isoDate: new Date(tx.invoiceTime).toISOString(), + isoDate, usdValue: tx.usdPrice, rawTx } diff --git a/src/partners/bitsofgold.ts b/src/partners/bitsofgold.ts index 6cf6b7b4..08256995 100644 --- a/src/partners/bitsofgold.ts +++ b/src/partners/bitsofgold.ts @@ -10,7 +10,6 @@ import { import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' const asBogTx = asObject({ attributes: asObject({ @@ -34,6 +33,7 @@ const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days export async function queryBitsOfGold( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const ssFormatTxs: StandardTx[] = [] let apiKey = '' let previousDate = '2019-01-01T00:00:00.000Z' @@ -69,7 +69,7 @@ export async function queryBitsOfGold( const response = await fetch(url, { method: 'GET', headers: headers }) result = asBogResult(await response.json()) } catch (e) { - datelog(e) + log.error(String(e)) throw e } const txs = result.data @@ -127,6 +127,9 @@ export function processBitsOfGoldTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount, direction, exchangeType: 'fiat', @@ -134,6 +137,9 @@ export function processBitsOfGoldTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount, timestamp: timestamp / 1000, isoDate: date.toISOString(), diff --git a/src/partners/bity.ts b/src/partners/bity.ts index 7fc4612d..f6ab82e2 100644 --- a/src/partners/bity.ts +++ b/src/partners/bity.ts @@ -2,7 +2,7 @@ import { asArray, asObject, asString, asUnknown } from 'cleaners' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, safeParseFloat } from '../util' +import { safeParseFloat } from '../util' const asBityTx = asObject({ id: asString, @@ -25,6 +25,7 @@ const PAGE_SIZE = 100 export async function queryBity( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let tokenParams let credentials @@ -111,7 +112,7 @@ export async function queryBity( break } } catch (e) { - datelog(e) + log.error(String(e)) throw e } @@ -177,6 +178,9 @@ export function processBityTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: tx.input.currency.toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: safeParseFloat(tx.input.amount), direction, exchangeType: 'fiat', @@ -184,6 +188,9 @@ export function processBityTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: tx.output.currency.toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat(tx.output.amount), timestamp: Date.parse(tx.timestamp_created.concat('Z')) / 1000, isoDate: new Date(tx.timestamp_created.concat('Z')).toISOString(), diff --git a/src/partners/changehero.ts b/src/partners/changehero.ts index 7ca5ccd9..369ffcd2 100644 --- a/src/partners/changehero.ts +++ b/src/partners/changehero.ts @@ -1,6 +1,8 @@ import { asArray, + asEither, asMaybe, + asNull, asNumber, asObject, asOptional, @@ -13,15 +15,13 @@ import { PartnerPlugin, PluginParams, PluginResult, + ScopedLog, StandardTx, Status } from '../types' -import { - datelog, - retryFetch, - safeParseFloat, - smartIsoDateFromTimestamp -} from '../util' +import { retryFetch, safeParseFloat, smartIsoDateFromTimestamp } from '../util' +import { createTokenId, EdgeTokenId, tokenTypes } from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' const asChangeHeroStatus = asMaybe(asValue('finished', 'expired'), 'other') @@ -36,7 +36,20 @@ const asChangeHeroTx = asObject({ payoutAddress: asString, currencyTo: asString, amountTo: asString, - createdAt: asNumber + createdAt: asNumber, + chainFrom: asOptional(asString), + chainTo: asOptional(asString) +}) + +// Cleaner for currency data from getCurrenciesFull API +const asChangeHeroCurrency = asObject({ + name: asString, // ticker + blockchain: asString, // chain name + contractAddress: asEither(asString, asNull) +}) + +const asChangeHeroCurrenciesResult = asObject({ + result: asArray(asChangeHeroCurrency) }) const asChangeHeroPluginParams = asObject({ @@ -56,18 +69,222 @@ type ChangeHeroTx = ReturnType type ChangeHeroStatus = ReturnType const API_URL = 'https://api.changehero.io/v2/' -const LIMIT = 100 +const LIMIT = 300 const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days +// Date after which chainFrom/chainTo fields are required in rawTx +// Transactions before this date are allowed to skip asset info backfill +// Based on database analysis: newest tx without chain fields was 2023-12-03 +const CHAIN_FIELDS_REQUIRED_DATE = '2023-12-04T00:00:00.000Z' + const statusMap: { [key in ChangeHeroStatus]: Status } = { finished: 'complete', expired: 'expired', other: 'other' } +// Map Changehero chain names to Edge pluginIds +const CHANGEHERO_CHAIN_TO_PLUGIN_ID: Record = { + algorand: 'algorand', + arbitrum: 'arbitrum', + avalanche: 'avalanche', + 'avalanche_(c-chain)': 'avalanche', + base: 'base', + binance: 'binance', + binance_smart_chain: 'binancesmartchain', + binancesmartchain: 'binancesmartchain', + bitcoin: 'bitcoin', + bitcoincash: 'bitcoincash', + bitcoinsv: 'bitcoinsv', + cardano: 'cardano', + cosmos: 'cosmoshub', + dash: 'dash', + digibyte: 'digibyte', + dogecoin: 'dogecoin', + ethereum: 'ethereum', + ethereumclassic: 'ethereumclassic', + hedera: 'hedera', + hypeevm: 'hyperevm', + litecoin: 'litecoin', + monero: 'monero', + optimism: 'optimism', + polkadot: 'polkadot', + polygon: 'polygon', + qtum: 'qtum', + ripple: 'ripple', + solana: 'solana', + stellar: 'stellar', + sui: 'sui', + tezos: 'tezos', + ton: 'ton', + tron: 'tron' +} + +// Cache for currency contract addresses: key = "TICKER_chain" -> contractAddress +interface CurrencyInfo { + contractAddress: string | null +} +let currencyCache: Map | null = null + +function makeCurrencyCacheKey(ticker: string, chain: string): string { + return `${ticker.toUpperCase()}_${chain.toLowerCase()}` +} + +// Hardcoded fallback for currencies not in getCurrenciesFull API +// Key format: "TICKER_chain" (uppercase ticker, lowercase chain) +const MISSING_CURRENCIES: Record = { + AVAX_avalanche: { contractAddress: null }, + BCH_bitcoincash: { contractAddress: null }, + BNB_binance: { contractAddress: null }, + BNB_binancesmartchain: { contractAddress: null }, + BSV_bitcoinsv: { contractAddress: null }, + BUSD_binance_smart_chain: { + contractAddress: '0xe9e7cea3dedca5984780bafc599bd69add087d56' + }, + BUSD_binancesmartchain: { + contractAddress: '0xe9e7cea3dedca5984780bafc599bd69add087d56' + }, + BUSD_ethereum: { + contractAddress: '0x4fabb145d64652a948d72533023f6e7a623c7c53' + }, + DOGE_dogecoin: { contractAddress: null }, + ETC_ethereumclassic: { contractAddress: null }, + FTM_ethereum: { + contractAddress: '0x4e15361fd6b4bb609fa63c81a2be19d873717870' + }, + GALA_ethereum: { + contractAddress: '0xd1d2eb1b1e90b638588728b4130137d262c87cae' + }, + KEY_ethereum: { + contractAddress: '0x4cc19356f2d37338b9802aa8e8fc58b0373296e7' + }, + MKR_ethereum: { + contractAddress: '0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2' + }, + OCEAN_ethereum: { + contractAddress: '0x967da4048cd07ab37855c090aaf366e4ce1b9f48' + }, + OMG_ethereum: { + contractAddress: '0xd26114cd6ee289accf82350c8d8487fedb8a0c07' + }, + USDC_binancesmartchain: { + contractAddress: '0x8ac76a51cc950d9822d68b83fe1ad97b32cd580d' + }, + USDT_binancesmartchain: { + contractAddress: '0x55d398326f99059ff775485246999027b3197955' + }, + XMR_monero: { contractAddress: null } +} + +async function fetchCurrencyCache( + apiKey: string, + log: ScopedLog +): Promise { + if (currencyCache != null) return + + try { + const response = await retryFetch(API_URL, { + headers: { 'Content-Type': 'application/json', 'api-key': apiKey }, + method: 'POST', + body: JSON.stringify({ + method: 'getCurrenciesFull', + params: {} + }) + }) + + if (!response.ok) { + throw new Error(`Failed to fetch currencies: ${response.status}`) + } + + const result = await response.json() + const currencies = asChangeHeroCurrenciesResult(result).result + + currencyCache = new Map() + for (const currency of currencies) { + const key = makeCurrencyCacheKey(currency.name, currency.blockchain) + currencyCache.set(key, { + contractAddress: currency.contractAddress + }) + } + + // Add hardcoded fallbacks for currencies not in API + for (const [key, info] of Object.entries(MISSING_CURRENCIES)) { + if (!currencyCache.has(key)) { + currencyCache.set(key, info) + } + } + + log(`Cached ${currencyCache.size} currency entries`) + } catch (e) { + log.error(`Failed to fetch currency cache: ${e}`) + throw e + } +} + +interface AssetInfo { + chainPluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId +} + +function getAssetInfo( + chain: string | undefined, + currencyCode: string, + isoDate: string +): AssetInfo { + const isBeforeCutoff = isoDate < CHAIN_FIELDS_REQUIRED_DATE + + // Get chainPluginId from chain - throw if unknown (unless before cutoff date) + if (chain == null) { + if (isBeforeCutoff) { + // Allow older transactions to skip asset info + return { chainPluginId: undefined, evmChainId: undefined, tokenId: null } + } + throw new Error(`Missing chain for currency ${currencyCode}`) + } + + const chainPluginId = CHANGEHERO_CHAIN_TO_PLUGIN_ID[chain] + if (chainPluginId == null) { + throw new Error( + `Unknown Changehero chain "${chain}" for currency ${currencyCode}. Add mapping to CHANGEHERO_CHAIN_TO_PLUGIN_ID.` + ) + } + + // Get evmChainId if this is an EVM chain + const evmChainId = EVM_CHAIN_IDS[chainPluginId] + + // Look up contract address from cache + let tokenId: EdgeTokenId = null + if (currencyCache == null) { + throw new Error('Currency cache not initialized') + } + const key = makeCurrencyCacheKey(currencyCode, chain) + const currencyInfo = currencyCache.get(key) + if (currencyInfo == null) { + throw new Error(`Currency info not found for ${currencyCode} on ${chain}`) + } + if (currencyInfo?.contractAddress != null) { + const tokenType = tokenTypes[chainPluginId] + if (tokenType == null) { + throw new Error( + `Unknown tokenType for chainPluginId "${chainPluginId}" (currency: ${currencyCode}, chain: ${chain}). Add tokenType to tokenTypes.` + ) + } + // createTokenId will throw if the chain doesn't support tokens + tokenId = createTokenId( + tokenType, + currencyCode, + currencyInfo.contractAddress + ) + } + + return { chainPluginId, evmChainId, tokenId } +} + export async function queryChangeHero( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asChangeHeroPluginParams(pluginParams) const { apiKey } = apiKeys let offset = 0 @@ -77,6 +294,9 @@ export async function queryChangeHero( return { settings: { latestIsoDate }, transactions: [] } } + // Fetch currency cache for contract address lookups + await fetchCurrencyCache(apiKey, log) + const standardTxs: StandardTx[] = [] let previousTimestamp = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK if (previousTimestamp < 0) previousTimestamp = 0 @@ -86,7 +306,7 @@ export async function queryChangeHero( let done = false while (!done) { let oldestIsoDate = '999999999999999999999999999999999999' - datelog(`Query changeHero offset: ${offset}`) + log(`Query offset: ${offset}`) const params = { id: '', @@ -107,7 +327,7 @@ export async function queryChangeHero( if (!response.ok) { const text = await response.text() - datelog(text) + log.error(text) throw new Error(text) } @@ -115,11 +335,11 @@ export async function queryChangeHero( const txs = asChangeHeroResult(result).result if (txs.length === 0) { - datelog(`ChangeHero done at offset ${offset}`) + log(`Done at offset ${offset}`) break } for (const rawTx of txs) { - const standardTx = processChangeHeroTx(rawTx) + const standardTx = await processChangeHeroTx(rawTx, pluginParams) standardTxs.push(standardTx) if (standardTx.isoDate > latestIsoDate) { @@ -129,17 +349,15 @@ export async function queryChangeHero( oldestIsoDate = standardTx.isoDate } if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `ChangeHero done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` - ) + log(`Done: date ${standardTx.isoDate} < ${previousLatestIsoDate}`) done = true } } - datelog(`Changehero oldestIsoDate ${oldestIsoDate}`) + log(`oldestIsoDate ${oldestIsoDate}`) offset += LIMIT } } catch (e) { - datelog(e) + log.error(String(e)) } const out = { settings: { @@ -158,8 +376,28 @@ export const changehero: PartnerPlugin = { pluginId: 'changehero' } -export function processChangeHeroTx(rawTx: unknown): StandardTx { +export async function processChangeHeroTx( + rawTx: unknown, + pluginParams: PluginParams +): Promise { const tx: ChangeHeroTx = asChangeHeroTx(rawTx) + const { log } = pluginParams + + // Ensure currency cache is populated (for backfill script usage) + if (currencyCache == null) { + const { apiKeys } = asChangeHeroPluginParams(pluginParams) + if (apiKeys.apiKey != null) { + await fetchCurrencyCache(apiKeys.apiKey, log) + } + } + + const isoDate = smartIsoDateFromTimestamp(tx.createdAt).isoDate + + // Get deposit asset info + const depositAsset = getAssetInfo(tx.chainFrom, tx.currencyFrom, isoDate) + + // Get payout asset info + const payoutAsset = getAssetInfo(tx.chainTo, tx.currencyTo, isoDate) const standardTx: StandardTx = { status: statusMap[tx.status], @@ -168,6 +406,9 @@ export function processChangeHeroTx(rawTx: unknown): StandardTx { depositTxid: tx.payinHash, depositAddress: tx.payinAddress, depositCurrency: tx.currencyFrom.toUpperCase(), + depositChainPluginId: depositAsset.chainPluginId, + depositEvmChainId: depositAsset.evmChainId, + depositTokenId: depositAsset.tokenId, depositAmount: safeParseFloat(tx.amountFrom), direction: null, exchangeType: 'swap', @@ -175,9 +416,12 @@ export function processChangeHeroTx(rawTx: unknown): StandardTx { payoutTxid: tx.payoutHash, payoutAddress: tx.payoutAddress, payoutCurrency: tx.currencyTo.toUpperCase(), + payoutChainPluginId: payoutAsset.chainPluginId, + payoutEvmChainId: payoutAsset.evmChainId, + payoutTokenId: payoutAsset.tokenId, payoutAmount: safeParseFloat(tx.amountTo), timestamp: tx.createdAt, - isoDate: smartIsoDateFromTimestamp(tx.createdAt).isoDate, + isoDate, usdValue: -1, rawTx } diff --git a/src/partners/changelly.ts b/src/partners/changelly.ts index a961ae2f..93a4a8d9 100644 --- a/src/partners/changelly.ts +++ b/src/partners/changelly.ts @@ -1,8 +1,14 @@ import Changelly from 'api-changelly/lib.js' import { asArray, asNumber, asObject, asString, asUnknown } from 'cleaners' -import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, safeParseFloat } from '../util' +import { + PartnerPlugin, + PluginParams, + PluginResult, + ScopedLog, + StandardTx +} from '../types' +import { safeParseFloat } from '../util' const asChangellyTx = asObject({ id: asString, @@ -36,7 +42,8 @@ async function getTransactionsPromised( offset: number, currencyFrom: string | undefined, address: string | undefined, - extraId: string | undefined + extraId: string | undefined, + log: ScopedLog ): Promise> { let promise let attempt = 1 @@ -64,7 +71,7 @@ async function getTransactionsPromised( promise = await Promise.race([changellyFetch, timeoutTest]) if (promise === 'ETIMEDOUT' && attempt <= MAX_ATTEMPTS) { - datelog(`Changelly request timed out. Retry attempt: ${attempt}`) + log.warn(`Request timed out. Retry attempt: ${attempt}`) attempt++ continue } @@ -76,6 +83,7 @@ async function getTransactionsPromised( export async function queryChangelly( pluginParams: PluginParams ): Promise { + const { log } = pluginParams let changellySDK let latestTimeStamp = 0 let offset = 0 @@ -114,18 +122,19 @@ export async function queryChangelly( let done = false try { while (!done) { - datelog(`Query changelly offset: ${offset}`) + log(`Query offset: ${offset}`) const result = await getTransactionsPromised( changellySDK, LIMIT, offset, undefined, undefined, - undefined + undefined, + log ) const txs = asChangellyResult(result).result if (txs.length === 0) { - datelog(`Changelly done at offset ${offset}`) + log(`Done at offset ${offset}`) firstAttempt = false break } @@ -141,10 +150,9 @@ export async function queryChangelly( !done && !firstAttempt ) { - datelog( - `Changelly done: date ${ - standardTx.timestamp - } < ${latestTimeStamp - QUERY_LOOKBACK}` + log( + `Done: date ${standardTx.timestamp} < ${latestTimeStamp - + QUERY_LOOKBACK}` ) done = true } @@ -153,7 +161,7 @@ export async function queryChangelly( offset += LIMIT } } catch (e) { - datelog(e) + log.error(String(e)) } const out = { settings: { latestTimeStamp: newLatestTimeStamp, firstAttempt, offset }, @@ -180,6 +188,9 @@ export function processChangellyTx(rawTx: unknown): StandardTx { depositTxid: tx.payinHash, depositAddress: tx.payinAddress, depositCurrency: tx.currencyFrom.toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: safeParseFloat(tx.amountFrom), direction: null, exchangeType: 'swap', @@ -187,6 +198,9 @@ export function processChangellyTx(rawTx: unknown): StandardTx { payoutTxid: tx.payoutHash, payoutAddress: tx.payoutAddress, payoutCurrency: tx.currencyTo.toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat(tx.amountTo), timestamp: tx.createdAt, isoDate: new Date(tx.createdAt * 1000).toISOString(), diff --git a/src/partners/changenow.ts b/src/partners/changenow.ts index 29d22107..58f7d6c7 100644 --- a/src/partners/changenow.ts +++ b/src/partners/changenow.ts @@ -10,14 +10,182 @@ import { } from 'cleaners' import { - asStandardPluginParams, PartnerPlugin, PluginParams, PluginResult, + ScopedLog, StandardTx, Status } from '../types' -import { datelog, retryFetch, snooze } from '../util' +import { retryFetch, snooze } from '../util' +import { + ChainNameToPluginIdMapping, + createTokenId, + EdgeTokenId, + tokenTypes +} from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +// Map ChangeNow network names to Edge pluginIds +const CHANGENOW_NETWORK_TO_PLUGIN_ID: ChainNameToPluginIdMapping = { + btc: 'bitcoin', + ltc: 'litecoin', + eth: 'ethereum', + xrp: 'ripple', + xmr: 'monero', + bch: 'bitcoincash', + doge: 'dogecoin', + xlm: 'stellar', + trx: 'tron', + bsc: 'binancesmartchain', + sol: 'solana', + ada: 'cardano', + matic: 'polygon', + arbitrum: 'arbitrum', + base: 'base', + hbar: 'hedera', + algo: 'algorand', + ton: 'ton', + sui: 'sui', + cchain: 'avalanche', + avaxc: 'avalanche', + zec: 'zcash', + osmo: 'osmosis', + etc: 'ethereumclassic', + fil: 'filecoin', + ftm: 'fantom', + xtz: 'tezos', + atom: 'cosmoshub', + dot: 'polkadot', + dash: 'dash', + dgb: 'digibyte', + rvn: 'ravencoin', + bsv: 'bitcoinsv', + pls: 'pulsechain', + zksync: 'zksync', + op: 'optimism', + optimism: 'optimism', + coreum: 'coreum', + xec: 'ecash', + pivx: 'pivx', + pulse: 'pulsechain', + sonic: 'sonic', + fio: 'fio', + qtum: 'qtum', + celo: 'celo', + one: 'harmony', + ethw: 'ethereumpow', + binance: 'binance', + bnb: 'binance', + firo: 'zcoin', + axl: 'axelar', + stx: 'stacks', + btg: 'bitcoingold', + rune: 'thorchain', + eos: 'eos', + grs: 'groestlcoin', + xchain: 'avalanchexchain', + vet: 'vechain', + waxp: 'wax', + theta: 'theta', + ebst: 'eboost', + vtc: 'vertcoin', + smart: 'smartcash', + xzc: 'zcoin', + kin: 'kin', + eurs: 'eurs', + noah: 'noah', + dgtx: 'dgtx', + ptoy: 'ptoy', + fct: 'factom' +} + +// Cleaner for ChangeNow currency API response +const asChangeNowCurrency = asObject({ + ticker: asString, + network: asString, + tokenContract: asOptional(asString), + legacyTicker: asOptional(asString) +}) + +const asChangeNowCurrencyArray = asArray(asChangeNowCurrency) + +type ChangeNowCurrency = ReturnType + +// In-memory cache for currency lookups +// Key format: "ticker:network" -> tokenContract +interface CurrencyCache { + currencies: Map // ticker:network -> tokenContract + loaded: boolean +} + +const currencyCache: CurrencyCache = { + currencies: new Map(), + loaded: false +} + +/** + * Fetch all currencies from ChangeNow API and populate the cache + */ +async function loadCurrencyCache( + log: ScopedLog, + apiKey?: string +): Promise { + if (currencyCache.loaded) { + return + } + + try { + // The exchange/currencies endpoint doesn't require authentication + const url = 'https://api.changenow.io/v2/exchange/currencies?active=true' + const response = await retryFetch(url, { + method: 'GET' + }) + + if (!response.ok) { + const text = await response.text() + throw new Error(`Failed to fetch currencies: ${text}`) + } + + const result = await response.json() + const currencies = asChangeNowCurrencyArray(result) + + for (const currency of currencies) { + const key = `${currency.ticker.toLowerCase()}:${currency.network.toLowerCase()}` + currencyCache.currencies.set(key, currency.tokenContract ?? null) + + // Also cache by legacyTicker if different from ticker + if ( + currency.legacyTicker != null && + currency.legacyTicker !== currency.ticker + ) { + const legacyKey = `${currency.legacyTicker.toLowerCase()}:${currency.network.toLowerCase()}` + currencyCache.currencies.set(legacyKey, currency.tokenContract ?? null) + } + } + + currencyCache.loaded = true + log(`Currency cache loaded with ${currencies.length} entries`) + } catch (e) { + log.error(`Error loading currency cache: ${e}`) + throw e + } +} + +/** + * Look up contract address from cache + */ +function getContractFromCache( + ticker: string, + network: string +): string | null | undefined { + const key = `${ticker.toLowerCase()}:${network.toLowerCase()}` + if (currencyCache.currencies.has(key)) { + return currencyCache.currencies.get(key) + } + // Return undefined if not in cache (different from null which means native token) + return undefined +} const asChangeNowStatus = asMaybe( asValue('finished', 'waiting', 'expired'), @@ -30,6 +198,7 @@ const asChangeNowTx = asObject({ status: asChangeNowStatus, payin: asObject({ currency: asString, + network: asString, address: asString, amount: asOptional(asNumber), expectedAmount: asOptional(asNumber), @@ -37,6 +206,7 @@ const asChangeNowTx = asObject({ }), payout: asObject({ currency: asString, + network: asString, address: asString, amount: asOptional(asNumber), expectedAmount: asOptional(asNumber), @@ -49,6 +219,18 @@ const asChangeNowResult = asObject({ exchanges: asArray(asUnknown) }) type ChangeNowTx = ReturnType type ChangeNowStatus = ReturnType +// Custom plugin params cleaner +const asChangeNowPluginParams = asObject({ + apiKeys: asObject({ + apiKey: asOptional(asString) + }), + settings: asObject({ + latestIsoDate: asOptional(asString, '1970-01-01T00:00:00.000Z') + }) +}) + +type ChangeNowPluginParams = ReturnType + const MAX_RETRIES = 5 const LIMIT = 200 const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days @@ -63,9 +245,10 @@ const statusMap: { [key in ChangeNowStatus]: Status } = { export const queryChangeNow = async ( pluginParams: PluginParams ): Promise => { - const { settings, apiKeys } = asStandardPluginParams(pluginParams) - const { apiKey } = apiKeys - let { latestIsoDate } = settings + const { log } = pluginParams + const cleanParams = asChangeNowPluginParams(pluginParams) + const { apiKey } = cleanParams.apiKeys + let { latestIsoDate } = cleanParams.settings if (apiKey == null) { return { settings: { latestIsoDate }, transactions: [] } @@ -91,7 +274,7 @@ export const queryChangeNow = async ( }) if (!response.ok) { const text = await response.text() - datelog(`Error in offset:${offset}`) + log.error(`Error in offset:${offset}`) throw new Error(text) } const result = await response.json() @@ -101,21 +284,21 @@ export const queryChangeNow = async ( break } for (const rawTx of txs) { - const standardTx = processChangeNowTx(rawTx) + const standardTx = await processChangeNowTx(rawTx, pluginParams) standardTxs.push(standardTx) if (standardTx.isoDate > latestIsoDate) { latestIsoDate = standardTx.isoDate } } - datelog(`ChangeNow offset ${offset} latestIsoDate ${latestIsoDate}`) + log(`offset ${offset} latestIsoDate ${latestIsoDate}`) offset += txs.length retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${5 * retry}s`) + log.warn(`Snoozing ${5 * retry}s`) await snooze(5000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -138,12 +321,90 @@ export const changenow: PartnerPlugin = { pluginId: 'changenow' } -export function processChangeNowTx(rawTx: unknown): StandardTx { +interface EdgeAssetInfo { + chainPluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId | undefined +} + +/** + * Get the Edge asset info for a given network and currency code. + * Uses the cached currency data from the ChangeNow API. + */ +function getAssetInfo(network: string, currencyCode: string): EdgeAssetInfo { + // Map network to pluginId + const chainPluginId = CHANGENOW_NETWORK_TO_PLUGIN_ID[network.toLowerCase()] + if (chainPluginId == null) { + throw new Error(`Unknown network: ${network}`) + } + + const evmChainId = EVM_CHAIN_IDS[chainPluginId] + + // Look up contract address from cache + const contractAddress = getContractFromCache(currencyCode, network) + + // If not in cache or no contract address, it's a native token + if (contractAddress == null) { + return { + chainPluginId, + evmChainId, + tokenId: null + } + } + + // Create tokenId from contract address + const tokenType = tokenTypes[chainPluginId] + if (tokenType == null) { + // Chain doesn't support tokens, but we have a contract address + // This shouldn't happen, but treat as native + return { + chainPluginId, + evmChainId, + tokenId: null + } + } + + try { + const tokenId = createTokenId( + tokenType, + currencyCode.toUpperCase(), + contractAddress + ) + return { + chainPluginId, + evmChainId, + tokenId + } + } catch (e) { + // If tokenId creation fails, treat as native (no log available in this sync function) + return { + chainPluginId, + evmChainId, + tokenId: null + } + } +} + +export async function processChangeNowTx( + rawTx: unknown, + pluginParams: PluginParams +): Promise { + const { log } = pluginParams + // Load currency cache before processing transactions + await loadCurrencyCache(log) + const tx: ChangeNowTx = asChangeNowTx(rawTx) const date = new Date( tx.createdAt.endsWith('Z') ? tx.createdAt : tx.createdAt + 'Z' ) const timestamp = date.getTime() / 1000 + + // Get deposit asset info + const depositAsset = getAssetInfo(tx.payin.network, tx.payin.currency) + + // Get payout asset info + const payoutAsset = getAssetInfo(tx.payout.network, tx.payout.currency) + const standardTx: StandardTx = { status: statusMap[tx.status], orderId: tx.requestId, @@ -151,6 +412,9 @@ export function processChangeNowTx(rawTx: unknown): StandardTx { depositTxid: tx.payin.hash, depositAddress: tx.payin.address, depositCurrency: tx.payin.currency.toUpperCase(), + depositChainPluginId: depositAsset.chainPluginId, + depositEvmChainId: depositAsset.evmChainId, + depositTokenId: depositAsset.tokenId, depositAmount: tx.payin.amount ?? tx.payin.expectedAmount ?? 0, direction: null, exchangeType: 'swap', @@ -158,6 +422,9 @@ export function processChangeNowTx(rawTx: unknown): StandardTx { payoutTxid: tx.payout.hash, payoutAddress: tx.payout.address, payoutCurrency: tx.payout.currency.toUpperCase(), + payoutChainPluginId: payoutAsset.chainPluginId, + payoutEvmChainId: payoutAsset.evmChainId, + payoutTokenId: payoutAsset.tokenId, payoutAmount: tx.payout.amount ?? tx.payout.expectedAmount ?? 0, timestamp, isoDate: date.toISOString(), diff --git a/src/partners/coinswitch.ts b/src/partners/coinswitch.ts index ea054a38..5d74204b 100644 --- a/src/partners/coinswitch.ts +++ b/src/partners/coinswitch.ts @@ -10,7 +10,6 @@ import { import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' import { queryDummy } from './dummy' const asCoinSwitchTx = asObject({ @@ -38,6 +37,7 @@ const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days export async function queryCoinSwitch( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let start = 0 let apiKey = '' @@ -68,7 +68,7 @@ export async function queryCoinSwitch( const result = await fetch(url, { method: 'GET', headers: headers }) jsonObj = asCoinSwitchResult(await result.json()) } catch (e) { - datelog(e) + log.error(String(e)) break } const txs = jsonObj.data.items @@ -117,6 +117,9 @@ export function processCoinSwitchTx(rawTx: unknown): StandardTx { depositTxid, depositAddress: tx.exchangeAddress.address, depositCurrency: tx.depositCoin.toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.depositCoinAmount, direction: null, exchangeType: 'swap', @@ -124,6 +127,9 @@ export function processCoinSwitchTx(rawTx: unknown): StandardTx { payoutTxid, payoutAddress: tx.destinationAddress.address, payoutCurrency: tx.destinationCoin.toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.destinationCoinAmount, timestamp: tx.createdAt / 1000, isoDate: new Date(tx.createdAt).toISOString(), diff --git a/src/partners/exolix.ts b/src/partners/exolix.ts index 782ddef3..41a8fc09 100644 --- a/src/partners/exolix.ts +++ b/src/partners/exolix.ts @@ -18,11 +18,16 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp } from '../util' +import { retryFetch, smartIsoDateFromTimestamp } from '../util' +import { createTokenId, EdgeTokenId, tokenTypes } from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +// Start date for Exolix transactions +const EXOLIX_START_DATE = '2020-01-01T00:00:00.000Z' const asExolixPluginParams = asObject({ settings: asObject({ - latestIsoDate: asOptional(asString, '0') + latestIsoDate: asOptional(asString, EXOLIX_START_DATE) }), apiKeys: asObject({ apiKey: asOptional(asString) @@ -46,10 +51,14 @@ const asExolixTx = asObject({ id: asString, status: asExolixStatus, coinFrom: asObject({ - coinCode: asString + coinCode: asString, + network: asOptional(asString), + contract: asMaybe(asEither(asString, asNull), null) }), coinTo: asObject({ - coinCode: asString + coinCode: asString, + network: asOptional(asString), + contract: asMaybe(asEither(asString, asNull), null) }), amount: asNumber, amountTo: asNumber, @@ -71,6 +80,12 @@ const asExolixResult = asObject({ const PAGE_LIMIT = 100 const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 3 // 3 days +// Date after which network fields are required in rawTx +// Transactions before this date are allowed to skip asset info backfill +// Based on database analysis: network fields started appearing around 2022-07-15 +// with the last transaction without network on 2022-08-12 +const NETWORK_FIELDS_REQUIRED_DATE = '2022-09-01T00:00:00.000Z' + type ExolixTx = ReturnType type ExolixStatus = ReturnType const statusMap: { [key in ExolixStatus]: Status } = { @@ -84,11 +99,133 @@ const statusMap: { [key in ExolixStatus]: Status } = { other: 'other' } +// Map Exolix network names to Edge pluginIds +const EXOLIX_NETWORK_TO_PLUGIN_ID: Record = { + ADA: 'cardano', + ALGO: 'algorand', + ARBITRUM: 'arbitrum', + ARRR: 'piratechain', + ATOM: 'cosmoshub', + AVAX: 'avalanche', + AVAXC: 'avalanche', + BASE: 'base', + BCH: 'bitcoincash', + BNB: 'binance', + BSC: 'binancesmartchain', + BTC: 'bitcoin', + BTG: 'bitcoingold', + CELO: 'celo', + DASH: 'dash', + DGB: 'digibyte', + DOGE: 'dogecoin', + DOT: 'polkadot', + EOS: 'eos', + ETC: 'ethereumclassic', + ETH: 'ethereum', + FIL: 'filecoin', + FIO: 'fio', + FTM: 'fantom', + HBAR: 'hedera', + HYPE: 'hyperevm', + LTC: 'litecoin', + MATIC: 'polygon', + OPTIMISM: 'optimism', + OSMO: 'osmosis', + PIVX: 'pivx', + POLYGON: 'polygon', + QTUM: 'qtum', + RUNE: 'thorchainrune', + RVN: 'ravencoin', + SOL: 'solana', + SUI: 'sui', + TELOS: 'telos', + TEZOS: 'tezos', + TON: 'ton', + TRX: 'tron', + XEC: 'ecash', + XLM: 'stellar', + XMR: 'monero', + XRP: 'ripple', + XTZ: 'tezos', + ZANO: 'zano', + ZEC: 'zcash', + ZKSYNCERA: 'zksync' +} + +// Contract addresses that represent native/gas tokens (not actual token contracts) +// These should be skipped when creating tokenId. Ideally providers should leave contracts empty for native tokens. +const GASTOKEN_CONTRACTS = [ + '0', // ALGO, TRX placeholder + '0x0d01dc56dcaaca66ad901c959b4011ec', // HYPE native + '0x1953cab0E5bFa6D4a9BaD6E05fD46C1CC6527a5a', // ETC wrapped + '0x471ece3750da237f93b8e339c536989b8978a438', // CELO native token + '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', // ETH native placeholder + 'EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c', // TON native + 'So11111111111111111111111111111111111111111', // SOL native wrapped + 'Tez', // XTZ native + 'lovelace', // ADA native unit + 'uosmo', // OSMO native denom + 'xrp' // XRP native +] + +interface AssetInfo { + chainPluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId +} + +function getAssetInfo( + network: string | undefined, + currencyCode: string, + contract: string | null, + isoDate: string +): AssetInfo { + const isBeforeCutoff = isoDate < NETWORK_FIELDS_REQUIRED_DATE + + // Get chainPluginId from network - throw if unknown (unless before cutoff date) + if (network == null) { + if (isBeforeCutoff) { + // Allow older transactions to skip asset info + return { chainPluginId: undefined, evmChainId: undefined, tokenId: null } + } + throw new Error(`Missing network for currency ${currencyCode}`) + } + + const chainPluginId = EXOLIX_NETWORK_TO_PLUGIN_ID[network] + if (chainPluginId == null) { + throw new Error( + `Unknown Exolix network "${network}" for currency ${currencyCode}. Add mapping to EXOLIX_NETWORK_TO_PLUGIN_ID.` + ) + } + + // Get evmChainId if this is an EVM chain + const evmChainId = EVM_CHAIN_IDS[chainPluginId] + + // Look up tokenId from contract address + let tokenId: EdgeTokenId = null + if (contract != null) { + if (GASTOKEN_CONTRACTS.includes(contract) && network === currencyCode) { + tokenId = null + } else { + const tokenType = tokenTypes[chainPluginId] + if (tokenType == null) { + throw new Error( + `Unknown tokenType for chainPluginId "${chainPluginId}" (currency: ${currencyCode}, network: ${network}, contract: ${contract}). Add tokenType to tokenTypes.` + ) + } + tokenId = createTokenId(tokenType, currencyCode, contract) + } + } + + return { chainPluginId, evmChainId, tokenId } +} + type Response = ReturnType export async function queryExolix( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asExolixPluginParams(pluginParams) const { apiKey } = apiKeys let { latestIsoDate } = settings @@ -105,48 +242,46 @@ export async function queryExolix( let done = false let page = 1 - while (!done) { - let oldestIsoDate = '999999999999999999999999999999999999' - let result - const request = `https://exolix.com/api/v2/transactions?page=${page}&size=${PAGE_LIMIT}` - const options = { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `${apiKey}` + try { + while (!done) { + const request = `https://exolix.com/api/v2/transactions?order=asc&page=${page}&size=${PAGE_LIMIT}&dateFrom=${previousLatestIsoDate}` + const options = { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `${apiKey}` + } } - } - const response = await retryFetch(request, options) - - if (response.ok) { - result = asExolixResult(await response.json()) - } + const response = await retryFetch(request, options) - const txs = result.data - for (const rawTx of txs) { - const standardTx = processExolixTx(rawTx) - standardTxs.push(standardTx) - if (standardTx.isoDate > latestIsoDate) { - latestIsoDate = standardTx.isoDate + if (!response.ok) { + const text = await response.text() + throw new Error(text) } - if (standardTx.isoDate < oldestIsoDate) { - oldestIsoDate = standardTx.isoDate + const json = await response.json() + const result = asExolixResult(json) + + const txs = result.data + for (const rawTx of txs) { + const standardTx = processExolixTx(rawTx) + standardTxs.push(standardTx) + if (standardTx.isoDate > latestIsoDate) { + latestIsoDate = standardTx.isoDate + } } - if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `Exolix done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` - ) + page++ + log(`latestIsoDate ${latestIsoDate}`) + + // reached end of database + if (txs.length < PAGE_LIMIT) { done = true } } - page++ - datelog(`Exolix oldestIsoDate ${oldestIsoDate}`) - - // reached end of database - if (txs.length < PAGE_LIMIT) { - done = true - } + } catch (e) { + log.error(String(e)) + // Do not throw as we can just exit and save our progress since the API allows querying + // from oldest to newest. } const out: PluginResult = { @@ -168,6 +303,23 @@ export function processExolixTx(rawTx: unknown): StandardTx { const tx: ExolixTx = asExolixTx(rawTx) const dateInMillis = Date.parse(tx.createdAt) const { isoDate, timestamp } = smartIsoDateFromTimestamp(dateInMillis) + + // Get deposit asset info + const depositAsset = getAssetInfo( + tx.coinFrom.network, + tx.coinFrom.coinCode, + tx.coinFrom.contract, + isoDate + ) + + // Get payout asset info + const payoutAsset = getAssetInfo( + tx.coinTo.network, + tx.coinTo.coinCode, + tx.coinTo.contract, + isoDate + ) + const standardTx: StandardTx = { status: statusMap[tx.status], orderId: tx.id, @@ -175,6 +327,9 @@ export function processExolixTx(rawTx: unknown): StandardTx { depositTxid: tx.hashIn?.hash ?? '', depositAddress: tx.depositAddress, depositCurrency: tx.coinFrom.coinCode, + depositChainPluginId: depositAsset.chainPluginId, + depositEvmChainId: depositAsset.evmChainId, + depositTokenId: depositAsset.tokenId, depositAmount: tx.amount, direction: null, exchangeType: 'swap', @@ -182,6 +337,9 @@ export function processExolixTx(rawTx: unknown): StandardTx { payoutTxid: tx.hashOut?.hash ?? '', payoutAddress: tx.withdrawalAddress, payoutCurrency: tx.coinTo.coinCode, + payoutChainPluginId: payoutAsset.chainPluginId, + payoutEvmChainId: payoutAsset.evmChainId, + payoutTokenId: payoutAsset.tokenId, payoutAmount: tx.amountTo, timestamp, isoDate, diff --git a/src/partners/faast.ts b/src/partners/faast.ts index e5f3666e..bf702833 100644 --- a/src/partners/faast.ts +++ b/src/partners/faast.ts @@ -3,7 +3,6 @@ import crypto from 'crypto' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' import { queryDummy } from './dummy' const asFaastTx = asObject({ @@ -33,6 +32,7 @@ const QUERY_LOOKBACK = 60 * 60 * 24 * 5 // 5 days export async function queryFaast( pluginParams: PluginParams ): Promise { + const { log } = pluginParams let page = 1 const standardTxs: StandardTx[] = [] let signature = '' @@ -70,7 +70,7 @@ export async function queryFaast( resultJSON = await result.json() jsonObj = asFaastResult(resultJSON) } catch (e) { - datelog(e) + log.error(String(e)) throw e } const txs = jsonObj.orders @@ -117,6 +117,9 @@ export function processFaastTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: tx.deposit_address, depositCurrency: tx.deposit_currency.toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.amount_deposited, direction: null, exchangeType: 'swap', @@ -124,6 +127,9 @@ export function processFaastTx(rawTx: unknown): StandardTx { payoutTxid: tx.transaction_id, payoutAddress: tx.withdrawal_address, payoutCurrency: tx.withdrawal_currency.toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.amount_withdrawn, timestamp, isoDate: tx.created_at, diff --git a/src/partners/foxExchange.ts b/src/partners/foxExchange.ts index 419594ab..4929bd72 100644 --- a/src/partners/foxExchange.ts +++ b/src/partners/foxExchange.ts @@ -2,7 +2,6 @@ import { asArray, asNumber, asObject, asString, asUnknown } from 'cleaners' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' import { queryDummy } from './dummy' const asFoxExchangeTx = asObject({ @@ -32,6 +31,7 @@ const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 3 // 3 days ago export async function queryFoxExchange( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let apiKey let secretToken @@ -77,7 +77,7 @@ export async function queryFoxExchange( txs = asFoxExchangeTxs(await res.json()) } } catch (e) { - datelog(e) + log.error(String(e)) throw e } @@ -119,13 +119,7 @@ export const foxExchange: PartnerPlugin = { } export function processFoxExchangeTx(rawTx: unknown): StandardTx { - let tx - try { - tx = asFoxExchangeTx(rawTx) - } catch (e) { - datelog(e) - throw e - } + const tx = asFoxExchangeTx(rawTx) const standardTx: StandardTx = { status: 'complete', orderId: tx.orderId, @@ -133,13 +127,19 @@ export function processFoxExchangeTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: tx.exchangeAddress.address, depositCurrency: tx.depositCoin.toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.depositCoinAmount, direction: null, exchangeType: 'swap', paymentType: null, - payoutTxid: tx.outputTransactionHash, + payoutTxid: undefined, payoutAddress: tx.destinationAddress.address, payoutCurrency: tx.destinationCoin.toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.destinationCoinAmount, timestamp: tx.createdAt / 1000, isoDate: new Date(tx.createdAt).toISOString(), diff --git a/src/partners/gebo.ts b/src/partners/gebo.ts index 471bdaa1..773c646a 100644 --- a/src/partners/gebo.ts +++ b/src/partners/gebo.ts @@ -1,12 +1,12 @@ import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' import { queryDummy } from './dummy' export async function queryGebo( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const ssFormatTxs: StandardTx[] = [] - await datelog('Running Gebo') + log('Running') return { settings: {}, transactions: ssFormatTxs diff --git a/src/partners/godex.ts b/src/partners/godex.ts index c4ed225c..ee5e1d2b 100644 --- a/src/partners/godex.ts +++ b/src/partners/godex.ts @@ -12,15 +12,18 @@ import { PartnerPlugin, PluginParams, PluginResult, + ScopedLog, StandardTx, Status } from '../types' +import { retryFetch, safeParseFloat, smartIsoDateFromTimestamp } from '../util' import { - datelog, - retryFetch, - safeParseFloat, - smartIsoDateFromTimestamp -} from '../util' + ChainNameToPluginIdMapping, + createTokenId, + EdgeTokenId, + tokenTypes +} from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' const asGodexPluginParams = asObject({ settings: asObject({ @@ -31,6 +34,201 @@ const asGodexPluginParams = asObject({ }) }) +// Godex started providing network_from_code/network_to_code fields on this date +// Transactions before this date are not required to have network codes +const GODEX_NETWORK_CODE_START_DATE = '2024-04-03T12:00:00.000Z' + +// Godex network codes to Edge pluginIds +const GODEX_NETWORK_TO_PLUGINID: ChainNameToPluginIdMapping = { + ADA: 'cardano', + ALGO: 'algorand', + ARBITRUM: 'arbitrum', + ATOM: 'cosmoshub', + AVAXC: 'avalanche', + BASE: 'base', + BCH: 'bitcoincash', + BNB: 'binancesmartchain', + BSC: 'binancesmartchain', + BTC: 'bitcoin', + BTG: 'bitcoingold', + CELO: 'celo', + DASH: 'dash', + DGB: 'digibyte', + DOGE: 'dogecoin', + DOT: 'polkadot', + EOS: 'eos', + ETC: 'ethereumclassic', + ETH: 'ethereum', + ETHW: 'ethereumpow', + FIL: 'filecoin', + FIO: 'fio', + FIRO: 'zcoin', + FTM: 'fantom', + HBAR: 'hedera', + LTC: 'litecoin', + MATIC: 'polygon', + OP: 'optimism', + OPTIMISM: 'optimism', + OSMO: 'osmosis', + PIVX: 'pivx', + QTUM: 'qtum', + RSK: 'rsk', + RUNE: 'thorchainrune', + RVN: 'ravencoin', + SOL: 'solana', + SUI: 'sui', + TON: 'ton', + TRX: 'tron', + XEC: 'ecash', + XLM: 'stellar', + XMR: 'monero', + XRP: 'ripple', + XTZ: 'tezos', + ZEC: 'zcash', + ZKSYNC: 'zksync' +} + +// Fallback for tokens that were delisted from Godex API but have historical transactions +const DELISTED_TOKENS: Record = { + 'TNSR:SOL': { contractAddress: 'TNSRxcUxoT9xBG3de7PiJyTDYu7kskLqcpddxnEJAS6' } +} + +// Cleaner for Godex coins API response +const asGodexCoinNetwork = asObject({ + code: asString, + contract_address: asOptional(asString), + chain_id: asOptional(asString) +}) + +const asGodexCoin = asObject({ + code: asString, + networks: asArray(asGodexCoinNetwork) +}) + +const asGodexCoinsResponse = asArray(asGodexCoin) + +// Cache for Godex coins data +interface GodexAssetInfo { + contractAddress?: string + chainId?: number +} + +let godexCoinsCache: Map | null = null + +async function getGodexCoinsCache( + log: ScopedLog +): Promise> { + if (godexCoinsCache != null) { + return godexCoinsCache + } + + const cache = new Map() + + // Add delisted tokens first (can be overwritten by API if re-listed) + for (const [key, value] of Object.entries(DELISTED_TOKENS)) { + cache.set(key, value) + } + + try { + const url = 'https://api.godex.io/api/v1/coins' + const result = await retryFetch(url, { method: 'GET' }) + const json = await result.json() + const coins = asGodexCoinsResponse(json) + + for (const coin of coins) { + for (const network of coin.networks) { + // Key format: "COIN_CODE:NETWORK_CODE" e.g. "USDT:TRX" + const key = `${coin.code}:${network.code}` + cache.set(key, { + contractAddress: network.contract_address ?? undefined, + chainId: + network.chain_id != null + ? parseInt(network.chain_id, 10) + : undefined + }) + } + } + log(`Coins cache loaded: ${cache.size} entries`) + } catch (e) { + log.error('Error loading coins cache:', e) + } + godexCoinsCache = cache + return cache +} + +interface GodexEdgeAssetInfo { + pluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId | undefined +} + +async function getGodexEdgeAssetInfo( + currencyCode: string, + networkCode: string | undefined, + isoDate: string, + log: ScopedLog +): Promise { + const result: GodexEdgeAssetInfo = { + pluginId: undefined, + evmChainId: undefined, + tokenId: undefined + } + + if (networkCode == null) { + // Only throw for transactions on or after the date when Godex started providing network codes + if (isoDate >= GODEX_NETWORK_CODE_START_DATE) { + throw new Error(`Godex: Missing network code for ${currencyCode}`) + } + // Older transactions without network codes cannot be backfilled + return result + } + + // Get pluginId from network code + const pluginId = GODEX_NETWORK_TO_PLUGINID[networkCode] + if (pluginId == null) { + throw new Error( + `Godex: Unknown network code '${networkCode}' for ${currencyCode}` + ) + } + result.pluginId = pluginId + + // Get evmChainId if applicable + result.evmChainId = EVM_CHAIN_IDS[pluginId] + + // Get contract address from cache + const cache = await getGodexCoinsCache(log) + const key = `${currencyCode}:${networkCode}` + const assetInfo = cache.get(key) + + if (assetInfo == null) { + // Some native coins (like SOL) aren't in Godex's coins API + // If currencyCode matches networkCode, assume it's a native coin + if (currencyCode === networkCode) { + result.tokenId = null + return result + } + throw new Error( + `Godex: Unknown currency code '${currencyCode}' for ${networkCode}` + ) + } + + // Determine tokenId + const tokenType = tokenTypes[pluginId] + const contractAddress = assetInfo.contractAddress + + // For native assets (no contract address), tokenId is null + // For tokens, use createTokenId + if (contractAddress != null && contractAddress !== '') { + // createTokenId will throw if token not supported on this chain + result.tokenId = createTokenId(tokenType, currencyCode, contractAddress) + } else { + // Native asset, tokenId is null + result.tokenId = null + } + + return result +} + const asGodexStatus = asMaybe( asValue( 'success', @@ -54,7 +252,9 @@ const asGodexTx = asObject({ withdrawal: asString, coin_to: asString, withdrawal_amount: asString, - created_at: asString + created_at: asString, + network_from_code: asOptional(asString), + network_to_code: asOptional(asString) }) const asGodexResult = asArray(asUnknown) @@ -77,6 +277,7 @@ const statusMap: { [key in GodexStatus]: Status } = { export async function queryGodex( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asGodexPluginParams(pluginParams) const { apiKey } = apiKeys let { latestIsoDate } = settings @@ -107,7 +308,7 @@ export async function queryGodex( const txs = asGodexResult(resultJSON) for (const rawTx of txs) { - const standardTx = processGodexTx(rawTx) + const standardTx = await processGodexTx(rawTx, pluginParams) standardTxs.push(standardTx) if (standardTx.isoDate > latestIsoDate) { latestIsoDate = standardTx.isoDate @@ -116,13 +317,11 @@ export async function queryGodex( oldestIsoDate = standardTx.isoDate } if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `Godex done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` - ) + log(`done: date ${standardTx.isoDate} < ${previousLatestIsoDate}`) done = true } } - datelog(`godex oldestIsoDate ${oldestIsoDate}`) + log(`oldestIsoDate ${oldestIsoDate}`) offset += LIMIT // this is if the end of the database is reached @@ -131,7 +330,7 @@ export async function queryGodex( } } } catch (e) { - datelog(e) + log.error(String(e)) throw e } const out: PluginResult = { @@ -148,24 +347,57 @@ export const godex: PartnerPlugin = { pluginId: 'godex' } -export function processGodexTx(rawTx: unknown): StandardTx { +export async function processGodexTx( + rawTx: unknown, + pluginParams: PluginParams +): Promise { + const { log } = pluginParams const tx: GodexTx = asGodexTx(rawTx) const ts = parseInt(tx.created_at) const { isoDate, timestamp } = smartIsoDateFromTimestamp(ts) + + // Extract network codes from tx + const networkFromCode = tx.network_from_code + const networkToCode = tx.network_to_code + + // Get deposit asset info + const depositCurrency = tx.coin_from.toUpperCase() + const depositAssetInfo = await getGodexEdgeAssetInfo( + depositCurrency, + networkFromCode, + isoDate, + log + ) + + // Get payout asset info + const payoutCurrency = tx.coin_to.toUpperCase() + const payoutAssetInfo = await getGodexEdgeAssetInfo( + payoutCurrency, + networkToCode, + isoDate, + log + ) + const standardTx: StandardTx = { status: statusMap[tx.status], orderId: tx.transaction_id, countryCode: null, depositTxid: tx.hash_in, depositAddress: tx.deposit, - depositCurrency: tx.coin_from.toUpperCase(), + depositCurrency, + depositChainPluginId: depositAssetInfo.pluginId, + depositEvmChainId: depositAssetInfo.evmChainId, + depositTokenId: depositAssetInfo.tokenId, depositAmount: safeParseFloat(tx.deposit_amount), direction: null, exchangeType: 'swap', paymentType: null, payoutTxid: undefined, payoutAddress: tx.withdrawal, - payoutCurrency: tx.coin_to.toUpperCase(), + payoutCurrency, + payoutChainPluginId: payoutAssetInfo.pluginId, + payoutEvmChainId: payoutAssetInfo.evmChainId, + payoutTokenId: payoutAssetInfo.tokenId, payoutAmount: safeParseFloat(tx.withdrawal_amount), timestamp, isoDate, diff --git a/src/partners/ioniagiftcard.ts b/src/partners/ioniagiftcard.ts index 64d578f6..a5dbf2d8 100644 --- a/src/partners/ioniagiftcard.ts +++ b/src/partners/ioniagiftcard.ts @@ -16,7 +16,7 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' import { queryDummy } from './dummy' const asIoniaStatus = asMaybe(asValue('complete'), 'other') @@ -53,6 +53,7 @@ const statusMap: { [key in IoniaStatus]: Status } = { export const queryIoniaGiftCards = async ( pluginParams: PluginParams ): Promise => { + const { log } = pluginParams const { settings, apiKeys } = asStandardPluginParams(pluginParams) const { apiKey } = apiKeys let { latestIsoDate } = settings @@ -86,7 +87,7 @@ export const queryIoniaGiftCards = async ( }) if (!response.ok) { const text = await response.text() - datelog(`Error in page:${page}`) + log.error(`Error in page:${page}`) throw new Error(text) } const result = await response.json() @@ -102,18 +103,18 @@ export const queryIoniaGiftCards = async ( latestIsoDate = standardTx.isoDate } } - datelog(`IoniaGiftCards latestIsoDate ${latestIsoDate}`) + log(`latestIsoDate ${latestIsoDate}`) page++ if (txs.length < LIMIT) { break } retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${5 * retry}s`) + log.warn(`Snoozing ${5 * retry}s`) await snooze(5000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -146,6 +147,9 @@ export function processIoniaGiftCardsTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: 'USD', + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.USDPaidByCustomer, direction: 'sell', exchangeType: 'fiat', @@ -153,6 +157,9 @@ export function processIoniaGiftCardsTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: 'USD', + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.GiftCardFaceValue, timestamp, isoDate, diff --git a/src/partners/ioniavisarewards.ts b/src/partners/ioniavisarewards.ts index 1eec421f..98eaab07 100644 --- a/src/partners/ioniavisarewards.ts +++ b/src/partners/ioniavisarewards.ts @@ -16,7 +16,7 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' import { queryDummy } from './dummy' const asIoniaStatus = asMaybe(asValue('complete'), 'other') @@ -53,6 +53,7 @@ const statusMap: { [key in IoniaStatus]: Status } = { export const queryIoniaVisaRewards = async ( pluginParams: PluginParams ): Promise => { + const { log } = pluginParams const { settings, apiKeys } = asStandardPluginParams(pluginParams) const { apiKey } = apiKeys let { latestIsoDate } = settings @@ -86,7 +87,7 @@ export const queryIoniaVisaRewards = async ( }) if (!response.ok) { const text = await response.text() - datelog(`Error in page:${page}`) + log.error(`Error in page:${page}`) throw new Error(text) } const result = await response.json() @@ -102,18 +103,18 @@ export const queryIoniaVisaRewards = async ( latestIsoDate = standardTx.isoDate } } - datelog(`IoniaVisaRewards latestIsoDate ${latestIsoDate}`) + log(`latestIsoDate ${latestIsoDate}`) page++ if (txs.length < LIMIT) { break } retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${5 * retry}s`) + log.warn(`Snoozing ${5 * retry}s`) await snooze(5000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -146,6 +147,9 @@ export function processIoniaVisaRewardsTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: 'USD', + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.USDPaidByCustomer, direction: 'sell', exchangeType: 'fiat', @@ -153,6 +157,9 @@ export function processIoniaVisaRewardsTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: 'USD', + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.GiftCardFaceValue, timestamp, isoDate, diff --git a/src/partners/kado.ts b/src/partners/kado.ts index 60a5250a..0cd37d5f 100644 --- a/src/partners/kado.ts +++ b/src/partners/kado.ts @@ -18,7 +18,7 @@ import { PluginResult, StandardTx } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' import { queryDummy } from './dummy' // Define cleaner for individual transactions in onRamps and offRamps @@ -66,6 +66,7 @@ const MAX_RETRIES = 5 export async function queryKado( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asStandardPluginParams(pluginParams) const { apiKey } = apiKeys let { latestIsoDate } = settings @@ -97,14 +98,14 @@ export async function queryKado( const standardTx: StandardTx = processKadoTx(rawTx) standardTxs.push(standardTx) } - datelog(`Kado latestIsoDate:${latestIsoDate}`) + log(`latestIsoDate:${latestIsoDate}`) retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${60 * retry}s`) + log(`Snoozing ${60 * retry}s`) await snooze(61000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -138,6 +139,9 @@ export function processKadoTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: 'USD', + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.paidAmountUsd, direction: tx.type, exchangeType: 'fiat', @@ -145,6 +149,9 @@ export function processKadoTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: tx.walletAddress, payoutCurrency: tx.cryptoCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.receiveUnitCount, timestamp, isoDate, @@ -159,6 +166,9 @@ export function processKadoTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: tx.cryptoCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.depositUnitCount, direction: tx.type, exchangeType: 'fiat', @@ -166,6 +176,9 @@ export function processKadoTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: 'USD', + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.receiveUsd, timestamp, isoDate, diff --git a/src/partners/letsexchange.ts b/src/partners/letsexchange.ts index fce878c6..620d4d4f 100644 --- a/src/partners/letsexchange.ts +++ b/src/partners/letsexchange.ts @@ -1,6 +1,9 @@ import { asArray, + asEither, asMaybe, + asNull, + asNumber, asObject, asOptional, asString, @@ -9,43 +12,57 @@ import { } from 'cleaners' import { - EDGE_APP_START_DATE, PartnerPlugin, PluginParams, PluginResult, + ScopedLog, StandardTx, Status } from '../types' -import { - datelog, - retryFetch, - safeParseFloat, - smartIsoDateFromTimestamp -} from '../util' +import { retryFetch, safeParseFloat, snooze } from '../util' +import { createTokenId, EdgeTokenId, tokenTypes } from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +const MAX_RETRIES = 5 +const QUERY_INTERVAL_MS = 1000 * 60 * 60 * 24 * 30 // 30 days in milliseconds +const LETSEXCHANGE_START_DATE = '2022-02-01T00:00:00.000Z' + +/** + * Max number of new transactions to save. This is to prevent overloading the db + * write and potentially causing a timeout or failure. The query will be retried + * starting from where it left off. + */ +const MAX_NEW_TRANSACTIONS = 20000 export const asLetsExchangePluginParams = asObject({ settings: asObject({ - latestIsoDate: asOptional(asString, EDGE_APP_START_DATE) + latestIsoDate: asOptional(asString, LETSEXCHANGE_START_DATE) }), apiKeys: asObject({ - affiliateId: asOptional(asString), - apiKey: asOptional(asString) + affiliateId: asString, + apiKey: asString }) }) -const asLetsExchangeStatus = asMaybe( - asValue( - 'success', - 'wait', - 'overdue', - 'refund', - 'exchanging', - 'sending_confirmation', - 'other' - ), - 'other' +const asLetsExchangeStatus = asValue( + 'wait', + 'confirmation', + 'confirmed', + 'exchanging', + 'overdue', + 'refund', + 'sending', + 'transferring', + 'sending_confirmation', + 'success', + 'aml_check_failed', + 'overdue', + 'error', + 'canceled', + 'refund' ) +// Cleaner for the new v2 API response const asLetsExchangeTx = asObject({ status: asLetsExchangeStatus, transaction_id: asString, @@ -56,92 +73,368 @@ const asLetsExchangeTx = asObject({ withdrawal: asString, coin_to: asString, withdrawal_amount: asString, - created_at: asString + created_at: asString, + // Older network fields from v1 API + network_from_code: asOptional(asEither(asString, asNull), null), + network_to_code: asOptional(asEither(asString, asNull), null), + // Network fields for asset info from v2 API + coin_from_network: asOptional(asEither(asString, asNull), null), + coin_to_network: asOptional(asEither(asString, asNull), null), + // Contract addresses from v2 API + coin_from_contract_address: asOptional(asEither(asString, asNull), null), + coin_to_contract_address: asOptional(asEither(asString, asNull), null) }) -const asLetsExchangeResult = asObject({ +// Pagination response from v2 API +const asLetsExchangeV2Result = asObject({ + current_page: asNumber, + last_page: asNumber, data: asArray(asUnknown) }) -type LetsExchangeTx = ReturnType +// Cleaner for coins API response +const asLetsExchangeCoin = asObject({ + code: asString, + network_code: asString, + contract_address: asEither(asString, asNull), + chain_id: asEither(asString, asNull) +}) + +const asLetsExchangeCoinsResult = asArray(asUnknown) + +type LetsExchangeTxV2 = ReturnType type LetsExchangeStatus = ReturnType -const LIMIT = 100 -const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days +const LIMIT = 1000 + +// Date when LetsExchange API started providing coin_from_network/coin_to_network fields. +// Based on direct API testing (Dec 2024), network fields are available for all +// transactions back to approximately Feb 23, 2022. +// Transactions before this date may be missing network fields and won't be backfilled. +// Transactions on or after this date MUST have network fields or will throw. +const NETWORK_FIELDS_AVAILABLE_DATE = '2022-02-24T00:00:00.000Z' + const statusMap: { [key in LetsExchangeStatus]: Status } = { - success: 'complete', wait: 'pending', + confirmation: 'confirming', + confirmed: 'processing', + exchanging: 'processing', overdue: 'expired', refund: 'refunded', - exchanging: 'processing', - sending_confirmation: 'other', - other: 'other' + sending: 'processing', + transferring: 'processing', + sending_confirmation: 'withdrawing', + success: 'complete', + aml_check_failed: 'blocked', + canceled: 'cancelled', + error: 'failed' +} + +// Map LetsExchange network codes to Edge pluginIds +// Values from coin_from_network / coin_to_network fields +const LETSEXCHANGE_NETWORK_TO_PLUGIN_ID: Record = { + ADA: 'cardano', + ALGO: 'algorand', + ARBITRUM: 'arbitrum', + ARRR: 'piratechain', + ATOM: 'cosmoshub', + AVAX: 'avalanche', + AVAXC: 'avalanche', + BASE: 'base', + BCH: 'bitcoincash', + BEP2: 'binance', + BEP20: 'binancesmartchain', + BNB: 'binancesmartchain', + BSV: 'bitcoinsv', + BTC: 'bitcoin', + BTG: 'bitcoingold', + CELO: 'celo', + CORE: 'coreum', + COREUM: 'coreum', + CTXC: 'cortex', + DASH: 'dash', + DGB: 'digibyte', + DOGE: 'dogecoin', + DOT: 'polkadot', + EOS: 'eos', + ERC20: 'ethereum', + ETC: 'ethereumclassic', + ETH: 'ethereum', + ETHW: 'ethereumpow', + EVER: 'everscale', // Everscale - not supported by Edge + FIL: 'filecoin', + FIO: 'fio', + FIRO: 'zcoin', + FTM: 'fantom', + GRS: 'groestlcoin', + HBAR: 'hedera', + HYPEEVM: 'hyperevm', + LTC: 'litecoin', + MATIC: 'polygon', + OPTIMISM: 'optimism', + OSMO: 'osmosis', + PIVX: 'pivx', + QTUM: 'qtum', + PLS: 'pulsechain', + POL: 'polygon', + RSK: 'rsk', + RUNE: 'thorchainrune', + RVN: 'ravencoin', + SOL: 'solana', + SONIC: 'sonic', + SUI: 'sui', + TLOS: 'telos', + TON: 'ton', + TRC20: 'tron', + TRX: 'tron', + WAXL: 'axelar', + XEC: 'ecash', + XLM: 'stellar', + XMR: 'monero', + XRP: 'ripple', + XTZ: 'tezos', + ZANO: 'zano', + ZEC: 'zcash', + ZKSERA: 'zksync', + ZKSYNC: 'zksync' +} + +// Native token placeholder addresses that should be treated as null (native coin) +// All values should be lowercase for case-insensitive matching +const NATIVE_TOKEN_ADDRESSES = new Set([ + '0', // Native token placeholder + '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', // Common EVM native placeholder + 'so11111111111111111111111111111111111111111', // Wrapped SOL (treat as native) + 'arbieth', + 'cchain', + 'eosio.token', + 'hjgfhj', + 'matic', + 'pol', + 'xmr1' +]) + +// In-memory cache for currency contract addresses +// Key format: `${code}_${network_code}` (both lowercase) +interface CoinInfo { + contractAddress: string | null + chainId: string | null +} + +let coinCache: Map | null = null +let coinCacheApiKey: string | null = null + +async function fetchCoinCache(apiKey: string, log: ScopedLog): Promise { + if (coinCache != null && coinCacheApiKey === apiKey) { + return // Already cached + } + + log('Fetching coins for cache...') + + const response = await retryFetch( + 'https://api.letsexchange.io/api/v1/coins', + { + headers: { + Authorization: `Bearer ${apiKey}` + }, + method: 'GET' + } + ) + + if (!response.ok) { + const text = await response.text() + throw new Error(`Failed to fetch LetsExchange coins: ${text}`) + } + + const result = await response.json() + const coins = asLetsExchangeCoinsResult(result) + + coinCache = new Map() + for (const rawCoin of coins) { + try { + const coin = asLetsExchangeCoin(rawCoin) + // Create key from code and network_code (both lowercase) + const key = `${coin.code.toLowerCase()}_${coin.network_code.toLowerCase()}` + coinCache.set(key, { + contractAddress: coin.contract_address, + chainId: coin.chain_id + }) + } catch { + // Skip coins that don't match our cleaner + } + } + + coinCacheApiKey = apiKey + log(`Cached ${coinCache.size} coins`) +} + +interface AssetInfo { + chainPluginId: string + evmChainId: number | undefined + tokenId: EdgeTokenId +} + +function getAssetInfo( + initialNetwork: string | null, + currencyCode: string, + contractAddress: string | null, + log: ScopedLog +): AssetInfo | undefined { + let network = initialNetwork + if (network == null) { + // Try using the currencyCode as the network + network = currencyCode + log(`Using currencyCode as network: ${network}`) + } + + const networkUpper = network.toUpperCase() + const chainPluginId = LETSEXCHANGE_NETWORK_TO_PLUGIN_ID[networkUpper] + + if (chainPluginId == null) { + throw new Error( + `Unknown network "${initialNetwork}" for currency ${currencyCode}. Add mapping to LETSEXCHANGE_NETWORK_TO_PLUGIN_ID.` + ) + } + + // Get evmChainId if this is an EVM chain + const evmChainId = EVM_CHAIN_IDS[chainPluginId] + const tokenType = tokenTypes[chainPluginId] + + // Determine tokenId from the contract address in the response + let tokenId: EdgeTokenId = null + + if (contractAddress == null) { + // Try to lookup contract address from cache + const key = `${currencyCode.toLowerCase()}_${network.toLowerCase()}` + const coinInfo = coinCache?.get(key) + if (coinInfo != null) { + contractAddress = coinInfo.contractAddress + } else { + // Try appending the network to the currency code + const backupKey = `${currencyCode.toLowerCase()}-${network.toLowerCase()}_${network.toLowerCase()}` + const backupCoinInfo = coinCache?.get(backupKey) + if (backupCoinInfo != null) { + contractAddress = backupCoinInfo.contractAddress + } + } + } + + if ( + contractAddress != null && + !NATIVE_TOKEN_ADDRESSES.has(contractAddress.toLowerCase()) && + contractAddress.toLowerCase() !== currencyCode.toLowerCase() + ) { + if (tokenType == null) { + throw new Error( + `Unknown tokenType for chainPluginId "${chainPluginId}" (currency: ${currencyCode}, contract: ${contractAddress}). Add tokenType to tokenTypes.` + ) + } + // createTokenId will throw if the chain doesn't support tokens + tokenId = createTokenId(tokenType, currencyCode, contractAddress) + } + + return { chainPluginId, evmChainId, tokenId } } export async function queryLetsExchange( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asLetsExchangePluginParams(pluginParams) const { affiliateId, apiKey } = apiKeys let { latestIsoDate } = settings - // let latestIsoDate = '2023-01-04T19:36:46.000Z' if (apiKey == null || affiliateId == null) { return { settings: { latestIsoDate }, transactions: [] } } const standardTxs: StandardTx[] = [] - let previousTimestamp = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK - if (previousTimestamp < 0) previousTimestamp = 0 - const previousLatestIsoDate = new Date(previousTimestamp).toISOString() - let oldestIsoDate = '999999999999999999999999999999999999' - - let page = 0 - let done = false const headers = { Authorization: 'Bearer ' + apiKey } - try { + // Query from the saved date forward in 30-day chunks (oldest to newest) + let windowStart = new Date(latestIsoDate).getTime() - QUERY_INTERVAL_MS + const now = Date.now() + let done = false + let newTxStart: number = 0 + + // Outer loop: iterate over 30-day windows + while (windowStart < now && !done) { + const windowEnd = Math.min(windowStart + QUERY_INTERVAL_MS, now) + const startTimestamp = Math.floor(windowStart / 1000) + const endTimestamp = Math.floor(windowEnd / 1000) + + const windowStartIso = new Date(windowStart).toISOString() + const windowEndIso = new Date(windowEnd).toISOString() + log(`Querying ${windowStartIso} to ${windowEndIso}`) + + let page = 1 + let retry = 0 + + // Inner loop: paginate through results within this window while (!done) { - const url = `https://api.letsexchange.io/api/v1/affiliate/history/${affiliateId}?limit=${LIMIT}&page=${page}&types=0` + const url = `https://api.letsexchange.io/api/v2/transactions-list?limit=${LIMIT}&page=${page}&start_date=${startTimestamp}&end_date=${endTimestamp}` - const result = await retryFetch(url, { headers, method: 'GET' }) - if (!result.ok) { - const text = await result.text() - datelog(text) - throw new Error(text) - } - const resultJSON = await result.json() - const { data: txs } = asLetsExchangeResult(resultJSON) - - for (const rawTx of txs) { - const standardTx = processLetsExchangeTx(rawTx) - standardTxs.push(standardTx) - if (standardTx.isoDate > latestIsoDate) { - latestIsoDate = standardTx.isoDate + try { + const result = await retryFetch(url, { headers, method: 'GET' }) + if (!result.ok) { + const text = await result.text() + log.error(`error at page ${page}: ${text}`) + throw new Error(text) + } + const resultJSON = await result.json() + const resultData = asLetsExchangeV2Result(resultJSON) + const txs = resultData.data + const currentPage = resultData.current_page + const lastPage = resultData.last_page + + for (const rawTx of txs) { + const standardTx = await processLetsExchangeTx(rawTx, pluginParams) + standardTxs.push(standardTx) + if (standardTx.isoDate > latestIsoDate) { + if (newTxStart === 0) { + newTxStart = standardTxs.length + } + latestIsoDate = standardTx.isoDate + } } - if (standardTx.isoDate < oldestIsoDate) { - oldestIsoDate = standardTx.isoDate + + log(`page ${page}/${lastPage} latestIsoDate ${latestIsoDate}`) + + // Check if we've reached the last page for this window + if (currentPage >= lastPage || txs.length === 0) { + break } - if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `Godex done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` + + page++ + retry = 0 + if (standardTxs.length - newTxStart >= MAX_NEW_TRANSACTIONS) { + latestIsoDate = windowStartIso + log.warn( + `Max new transactions reached, saving progress at ${latestIsoDate}` ) done = true + break + } + } catch (e) { + log.error(String(e)) + // Retry a few times with time delay to prevent throttling + retry++ + if (retry <= MAX_RETRIES) { + log.warn(`Snoozing ${5 * retry}s`) + await snooze(5000 * retry) + } else { + // We can safely save our progress since we go from oldest to newest. + latestIsoDate = windowStartIso + done = true + log.error(`Max retries reached, saving progress at ${latestIsoDate}`) } - } - datelog(`letsexchange oldestIsoDate ${oldestIsoDate}`) - - page++ - // this is if the end of the database is reached - if (txs.length < LIMIT) { - done = true } } - } catch (e) { - datelog(e) - throw e + + // Move to the next 30-day window + windowStart = windowEnd } const out: PluginResult = { @@ -150,6 +443,7 @@ export async function queryLetsExchange( } return out } + export const letsexchange: PartnerPlugin = { // queryFunc will take PluginSettings as arg and return PluginResult queryFunc: queryLetsExchange, @@ -158,17 +452,58 @@ export const letsexchange: PartnerPlugin = { pluginId: 'letsexchange' } -export function processLetsExchangeTx(rawTx: unknown): StandardTx { - const tx: LetsExchangeTx = asLetsExchangeTx(rawTx) - const ts = parseInt(tx.created_at) - const { isoDate, timestamp } = smartIsoDateFromTimestamp(ts) +export async function processLetsExchangeTx( + rawTx: unknown, + pluginParams: PluginParams +): Promise { + const { apiKeys } = asLetsExchangePluginParams(pluginParams) + const { apiKey } = apiKeys + const { log } = pluginParams + + await fetchCoinCache(apiKey, log) + + const tx = asLetsExchangeTx(rawTx) + + // created_at is in format "2025-12-13 07:22:50" (UTC assumed) or UNIX timestamp (10 digits) + let date: Date + if (/^\d{10}$/.test(tx.created_at)) { + date = new Date(parseInt(tx.created_at) * 1000) + } else { + date = new Date(tx.created_at.replace(' ', 'T') + 'Z') + } + const timestamp = Math.floor(date.getTime() / 1000) + const isoDate = date.toISOString() + + // Get deposit asset info using contract address from API response + const depositAsset = getAssetInfo( + tx.coin_from_network ?? tx.network_from_code, + tx.coin_from, + tx.coin_from_contract_address, + log + ) + // Get payout asset info using contract address from API response + const payoutAsset = getAssetInfo( + tx.coin_to_network ?? tx.network_to_code, + tx.coin_to, + tx.coin_to_contract_address, + log + ) + + const status = statusMap[tx.status] + if (status == null) { + throw new Error(`Unknown LetsExchange status "${tx.status}"`) + } + const standardTx: StandardTx = { - status: statusMap[tx.status], + status, orderId: tx.transaction_id, countryCode: null, depositTxid: tx.hash_in, depositAddress: tx.deposit, depositCurrency: tx.coin_from.toUpperCase(), + depositChainPluginId: depositAsset?.chainPluginId, + depositEvmChainId: depositAsset?.evmChainId, + depositTokenId: depositAsset?.tokenId, depositAmount: safeParseFloat(tx.deposit_amount), direction: null, exchangeType: 'swap', @@ -176,6 +511,9 @@ export function processLetsExchangeTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: tx.withdrawal, payoutCurrency: tx.coin_to.toUpperCase(), + payoutChainPluginId: payoutAsset?.chainPluginId, + payoutEvmChainId: payoutAsset?.evmChainId, + payoutTokenId: payoutAsset?.tokenId, payoutAmount: safeParseFloat(tx.withdrawal_amount), timestamp, isoDate, diff --git a/src/partners/libertyx.ts b/src/partners/libertyx.ts index 97b55a16..560b11bc 100644 --- a/src/partners/libertyx.ts +++ b/src/partners/libertyx.ts @@ -9,7 +9,6 @@ import { import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' const asLibertyxTx = asObject({ all_transactions_usd_sum: asNumber, @@ -26,6 +25,7 @@ const INCOMPLETE_DAY_RANGE = 3 export async function queryLibertyx( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let apiKey let result @@ -41,7 +41,7 @@ export async function queryLibertyx( }) result = asLibertyxResult(await response.json()) } catch (e) { - datelog(e) + log.error(String(e)) throw e } } else { @@ -90,6 +90,9 @@ export function processLibertyxTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: 'USD', + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.all_transactions_usd_sum, direction: 'buy', exchangeType: 'fiat', @@ -97,6 +100,9 @@ export function processLibertyxTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: 'BTC', + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: 0, timestamp: timestamp, isoDate: date.toISOString(), diff --git a/src/partners/lifi.ts b/src/partners/lifi.ts index fd3ab40d..ac6942e5 100644 --- a/src/partners/lifi.ts +++ b/src/partners/lifi.ts @@ -18,17 +18,19 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { createTokenId, tokenTypes } from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS, REVERSE_EVM_CHAIN_IDS } from '../util/chainIds' const PLUGIN_START_DATE = '2023-01-01T00:00:00.000Z' const asStatuses = asMaybe(asValue('DONE'), 'other') const asToken = asObject({ - // address: asString, - // chainId: asNumber, + address: asOptional(asString), + chainId: asOptional(asNumber), symbol: asString, - decimals: asNumber + decimals: asNumber, // name: asString, - // coinKey: asString, + coinKey: asOptional(asString) // logoURI: asString, // priceUSD: asString }) @@ -36,7 +38,7 @@ const asToken = asObject({ const asTransaction = asObject({ txHash: asString, // txLink: asString, - // amount: asString, + amount: asString, token: asOptional(asToken), // chainId: asNumber, // gasPrice: asString, @@ -45,7 +47,7 @@ const asTransaction = asObject({ // gasAmount: asString, // gasAmountUSD: asString, amountUSD: asOptional(asString), - value: asString, + // value: asString, timestamp: asOptional(asNumber) }) @@ -70,6 +72,7 @@ const asTransfersResult = asObject({ transfers: asArray(asUnknown) }) +type Transfer = ReturnType type PartnerStatuses = ReturnType const MAX_RETRIES = 5 @@ -84,6 +87,7 @@ const statusMap: { [key in PartnerStatuses]: Status } = { export async function queryLifi( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asStandardPluginParams(pluginParams) const { apiKey } = apiKeys let { latestIsoDate } = settings @@ -116,7 +120,7 @@ export async function queryLifi( const jsonObj = await response.json() const transferResults = asTransfersResult(jsonObj) for (const rawTx of transferResults.transfers) { - const standardTx = processLifiTx(rawTx) + const standardTx = processLifiTx(rawTx, pluginParams) standardTxs.push(standardTx) if (standardTx.isoDate > latestIsoDate) { latestIsoDate = standardTx.isoDate @@ -124,19 +128,17 @@ export async function queryLifi( } const endDate = new Date(endTime) startTime = endTime - datelog( - `Lifi endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}` - ) + log(`endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}`) if (endTime > now) { break } retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${60 * retry}s`) + log.warn(`Snoozing ${60 * retry}s`) await snooze(60000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -159,8 +161,18 @@ export const lifi: PartnerPlugin = { pluginId: 'lifi' } -export function processLifiTx(rawTx: unknown): StandardTx { - const tx = asTransfer(rawTx) +export function processLifiTx( + rawTx: unknown, + pluginParams: PluginParams +): StandardTx { + const { log } = pluginParams + let tx: Transfer + try { + tx = asTransfer(rawTx) + } catch (e) { + log.error(String(e)) + throw e + } const txTimestamp = tx.receiving.timestamp ?? tx.sending.timestamp ?? 0 if (txTimestamp === 0) { throw new Error('No timestamp') @@ -172,29 +184,171 @@ export function processLifiTx(rawTx: unknown): StandardTx { if (depositToken == null || payoutToken == null) { throw new Error('Missing token details') } - const depositAmount = Number(tx.sending.value) / 10 ** depositToken.decimals - - const payoutAmount = Number(tx.receiving.value) / 10 ** payoutToken.decimals - - const standardTx: StandardTx = { - status: statusMap[tx.status], - orderId: tx.sending.txHash, - countryCode: null, - depositTxid: tx.sending.txHash, - depositAddress: undefined, - depositCurrency: depositToken.symbol, - depositAmount, - direction: null, - exchangeType: 'swap', - paymentType: null, - payoutTxid: undefined, - payoutAddress: tx.toAddress, - payoutCurrency: payoutToken.symbol, - payoutAmount, - timestamp, - isoDate, - usdValue: Number(tx.receiving.amountUSD ?? tx.sending.amountUSD ?? '-1'), - rawTx + const depositAmount = Number(tx.sending.amount) / 10 ** depositToken.decimals + const payoutAmount = Number(tx.receiving.amount) / 10 ** payoutToken.decimals + + // Get the currencCode of the gasToken as we'll use this to determine if this is + // a token swap. If there's not gasToken object, use the token object. + const depositChainCodeUnmapped = + tx.sending.gasToken?.coinKey ?? + tx.sending.gasToken?.symbol ?? + depositToken?.coinKey ?? + depositToken?.symbol + const payoutChainCodeUnmappped = + tx.receiving.gasToken?.coinKey ?? + tx.receiving.gasToken?.symbol ?? + payoutToken?.coinKey ?? + payoutToken?.symbol + + // For some reason, some gasToken like Solana are given as "wSOL", so map them to SOL + const depositChainCode = + TOKEN_CODE_MAPPINGS[depositChainCodeUnmapped ?? ''] ?? + depositChainCodeUnmapped + const payoutChainCode = + TOKEN_CODE_MAPPINGS[payoutChainCodeUnmappped ?? ''] ?? + payoutChainCodeUnmappped + + const depositTokenCode = + tx.sending.token?.coinKey ?? + tx.sending.token?.symbol ?? + tx.sending.gasToken?.coinKey ?? + tx.sending.gasToken?.symbol + const payoutTokenCode = + tx.receiving.token?.coinKey ?? + tx.receiving.token?.symbol ?? + tx.receiving.gasToken?.coinKey ?? + tx.receiving.gasToken?.symbol + + // If the token code and chain code match, this is a gas token so + // tokenId = null + const depositTokenAddress = + depositTokenCode !== depositChainCode ? depositToken?.address : null + const payoutTokenAddress = + payoutTokenCode !== payoutChainCode ? payoutToken?.address : null + + // Try to determine the EVM chain id from the token chain id. Lifi + // has chainIds for non-EVM chains like Solana so we have to filter them out. + let depositEvmChainId = + REVERSE_EVM_CHAIN_IDS[depositToken.chainId ?? 0] != null + ? depositToken.chainId + : undefined + let payoutEvmChainId = + REVERSE_EVM_CHAIN_IDS[payoutToken.chainId ?? 0] != null + ? payoutToken.chainId + : undefined + + // Determine the chain plugin id and token id. + // Try using the gas token code first, then chain id if we have one. + const depositChainPluginId = + REVERSE_EVM_CHAIN_IDS[depositEvmChainId ?? 0] ?? + MAINNET_CODE_TRANSCRIPTION[ + tx.sending.gasToken?.coinKey ?? tx.sending.gasToken?.symbol ?? '' + ] + const payoutChainPluginId = + REVERSE_EVM_CHAIN_IDS[payoutEvmChainId ?? 0] ?? + MAINNET_CODE_TRANSCRIPTION[ + tx.receiving.gasToken?.coinKey ?? tx.receiving.gasToken?.symbol ?? '' + ] + + if (depositChainPluginId == null || payoutChainPluginId == null) { + throw new Error('Missing chain plugin id') } - return standardTx + + // If we weren't able to determine an EVM chain id, try to get it from the + // chain plugin id. + depositEvmChainId = + depositEvmChainId == null + ? EVM_CHAIN_IDS[depositChainPluginId ?? ''] + : depositEvmChainId + payoutEvmChainId = + payoutEvmChainId == null + ? EVM_CHAIN_IDS[payoutChainPluginId ?? ''] + : payoutEvmChainId + + const depositTokenType = tokenTypes[depositChainPluginId ?? ''] + const payoutTokenType = tokenTypes[payoutChainPluginId ?? ''] + + if (depositTokenType == null || payoutTokenType == null) { + throw new Error('Missing token type') + } + + try { + const depositTokenId = createTokenId( + depositTokenType, + depositToken.symbol, + depositTokenAddress ?? undefined + ) + const payoutTokenId = createTokenId( + payoutTokenType, + payoutToken.symbol, + payoutTokenAddress ?? undefined + ) + + const standardTx: StandardTx = { + status: statusMap[tx.status], + orderId: tx.sending.txHash, + countryCode: null, + depositTxid: tx.sending.txHash, + depositAddress: undefined, + depositCurrency: depositToken.symbol, + depositChainPluginId, + depositEvmChainId, + depositTokenId, + depositAmount, + direction: null, + exchangeType: 'swap', + paymentType: null, + payoutTxid: tx.receiving.txHash, + payoutAddress: tx.toAddress, + payoutCurrency: payoutToken.symbol, + payoutChainPluginId, + payoutEvmChainId, + payoutTokenId, + payoutAmount, + timestamp, + isoDate, + usdValue: Number(tx.sending.amountUSD ?? tx.receiving.amountUSD ?? '-1'), + rawTx + } + if (statusMap[tx.status] === 'complete') { + const { orderId, depositCurrency, payoutCurrency } = standardTx + console.log( + `${orderId} ${depositCurrency} ${depositChainPluginId} ${depositEvmChainId} ${depositTokenId?.slice( + 0, + 6 + ) ?? + ''} ${depositAmount} -> ${payoutCurrency} ${payoutChainPluginId} ${payoutEvmChainId} ${payoutTokenId?.slice( + 0, + 6 + ) ?? ''} ${payoutAmount}` + ) + } + return standardTx + } catch (e) { + log.error(String(e)) + throw e + } +} + +const MAINNET_CODE_TRANSCRIPTION: Record = { + ARBITRUM: 'arbitrum', + AVAX: 'avalanche', + BNB: 'binancesmartchain', + CELO: 'celo', + ETH: 'ethereum', + FTM: 'fantom', + HYPE: 'hyperevm', + OP: 'optimism', + POL: 'polygon', + PLS: 'pulsechain', + RBTC: 'rsk', + SOL: 'solana', + wSOL: 'solana', + SUI: 'sui', + SONIC: 'sonic', + ZKSYNC: 'zksync' +} + +const TOKEN_CODE_MAPPINGS: Record = { + wSOL: 'SOL' } diff --git a/src/partners/moonpay.ts b/src/partners/moonpay.ts index 067743f2..076ea68b 100644 --- a/src/partners/moonpay.ts +++ b/src/partners/moonpay.ts @@ -17,17 +17,98 @@ import { PartnerPlugin, PluginParams, PluginResult, - StandardTx + StandardTx, + Status } from '../types' -import { datelog } from '../util' +import { createTokenId, EdgeTokenId, tokenTypes } from '../util/asEdgeTokenId' +import { + EVM_CHAIN_IDS, + REVERSE_EVM_CHAIN_IDS, + reverseEvmChainId +} from '../util/chainIds' + +interface EdgeAssetInfo { + chainPluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId +} + +type MoonpayCurrencyMetadata = ReturnType + +/** + * Process Moonpay currency metadata to extract Edge asset info + */ +function processMetadata( + metadata: MoonpayCurrencyMetadata | undefined, + currencyCode: string +): EdgeAssetInfo { + if (metadata == null) { + throw new Error(`Missing metadata for currency ${currencyCode}`) + } + + const networkCode = metadata.networkCode + const rawChainId = metadata.chainId + const chainIdNum = rawChainId != null ? parseInt(rawChainId, 10) : undefined + + // Determine chainPluginId from networkCode or chainId + const chainPluginId = + moonpayNetworkToPluginId(networkCode) ?? reverseEvmChainId(chainIdNum) + + // Determine evmChainId + let evmChainId: number | undefined + if (chainIdNum != null && REVERSE_EVM_CHAIN_IDS[chainIdNum] != null) { + evmChainId = chainIdNum + } else if (chainPluginId != null && EVM_CHAIN_IDS[chainPluginId] != null) { + evmChainId = EVM_CHAIN_IDS[chainPluginId] + } + + // Determine tokenId from contract address + // If we have a chainPluginId but no contract address, it's a native/mainnet gas token (tokenId = null) + // If we have a contract address, create the tokenId + let tokenId: EdgeTokenId = null + const contractAddress = metadata.contractAddress + if (chainPluginId != null) { + if ( + contractAddress != null && + contractAddress !== '0x0000000000000000000000000000000000000000' + ) { + const tokenType = tokenTypes[chainPluginId] + if (tokenType == null) { + throw new Error( + `Unknown tokenType for chainPluginId ${chainPluginId} (currency: ${currencyCode})` + ) + } + tokenId = createTokenId( + tokenType, + currencyCode.toUpperCase(), + contractAddress + ) + } else { + // Native/mainnet gas token - explicitly null + tokenId = null + } + } + + return { chainPluginId, evmChainId, tokenId } +} + +const asMoonpayCurrencyMetadata = asObject({ + chainId: asOptional(asString), + networkCode: asOptional(asString), + contractAddress: asOptional(asString) +}) const asMoonpayCurrency = asObject({ id: asString, type: asString, name: asString, - code: asString + code: asString, + metadata: asOptional(asMoonpayCurrencyMetadata) }) +// Unified cleaner that handles both buy and sell transactions +// Buy transactions have: paymentMethod, cryptoTransactionId, currency, walletAddress +// Sell transactions have: payoutMethod, depositHash, quoteCurrency const asMoonpayTx = asObject({ baseCurrency: asMoonpayCurrency, baseCurrencyAmount: asNumber, @@ -35,34 +116,30 @@ const asMoonpayTx = asObject({ cardType: asOptional(asValue('apple_pay', 'google_pay', 'card')), country: asString, createdAt: asDate, - cryptoTransactionId: asString, - currencyId: asString, - currency: asMoonpayCurrency, id: asString, + status: asString, + // Common amount field (used by both buy and sell) + quoteCurrencyAmount: asOptional(asNumber), + // Buy-specific fields + cryptoTransactionId: asOptional(asString), + currency: asOptional(asMoonpayCurrency), + walletAddress: asOptional(asString), paymentMethod: asOptional(asString), - quoteCurrencyAmount: asNumber, - walletAddress: asString -}) - -const asMoonpaySellTx = asObject({ - baseCurrency: asMoonpayCurrency, - baseCurrencyAmount: asNumber, - baseCurrencyId: asString, - country: asString, - createdAt: asDate, - depositHash: asString, - id: asString, - paymentMethod: asOptional(asString), - quoteCurrency: asMoonpayCurrency, - quoteCurrencyAmount: asNumber + // Sell-specific fields + depositHash: asOptional(asString), + quoteCurrency: asOptional(asMoonpayCurrency), + payoutMethod: asOptional(asString) }) type MoonpayTx = ReturnType -type MoonpaySellTx = ReturnType +type MoonpayStatus = MoonpayTx['status'] -const asPreMoonpayTx = asObject({ - status: asString -}) +// Map Moonpay status to Edge status +// Only 'completed' and 'pending' were found in 3 years of API data +const statusMap: Record = { + completed: 'complete', + pending: 'pending' +} const asMoonpayResult = asArray(asUnknown) @@ -73,6 +150,7 @@ const PER_REQUEST_LIMIT = 50 export async function queryMoonpay( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let headers @@ -103,7 +181,7 @@ export async function queryMoonpay( try { do { - console.log(`Querying Moonpay from ${queryIsoDate} to ${latestIsoDate}`) + log(`Querying from ${queryIsoDate} to ${latestIsoDate}`) let offset = 0 while (true) { @@ -115,17 +193,16 @@ export async function queryMoonpay( const txs = asMoonpayResult(await result.json()) for (const rawTx of txs) { - if (asPreMoonpayTx(rawTx).status === 'completed') { - const standardTx = processMoonpaySellTx(rawTx) - standardTxs.push(standardTx) - } + const standardTx = processTx(rawTx) + standardTxs.push(standardTx) } if (txs.length > 0) { - console.log( - `Moonpay sell txs ${txs.length}: ${JSON.stringify( - txs.slice(-1) - ).slice(0, 100)}` + log( + `sell txs ${txs.length}: ${JSON.stringify(txs.slice(-1)).slice( + 0, + 100 + )}` ) } @@ -148,16 +225,15 @@ export async function queryMoonpay( // in bulk update it throws an error for document update conflict because of this. for (const rawTx of txs) { - if (asPreMoonpayTx(rawTx).status === 'completed') { - const standardTx = processMoonpayTx(rawTx) - standardTxs.push(standardTx) - } + const standardTx = processTx(rawTx) + standardTxs.push(standardTx) } if (txs.length > 0) { - console.log( - `Moonpay buy txs ${txs.length}: ${JSON.stringify( - txs.slice(-1) - ).slice(0, 100)}` + log( + `buy txs ${txs.length}: ${JSON.stringify(txs.slice(-1)).slice( + 0, + 100 + )}` ) } @@ -174,9 +250,8 @@ export async function queryMoonpay( } while (isoNow > latestIsoDate) latestIsoDate = isoNow } catch (e) { - datelog(e) - console.log(`Moonpay error: ${e}`) - console.log(`Saving progress up until ${queryIsoDate}`) + log.error(`Error: ${e}`) + log(`Saving progress up until ${queryIsoDate}`) // Set the latestIsoDate to the queryIsoDate so that the next query will // query the same time range again since we had a failure in that time range @@ -198,29 +273,58 @@ export const moonpay: PartnerPlugin = { pluginId: 'moonpay' } -export function processMoonpayTx(rawTx: unknown): StandardTx { +export function processTx(rawTx: unknown): StandardTx { const tx: MoonpayTx = asMoonpayTx(rawTx) const isoDate = tx.createdAt.toISOString() const timestamp = tx.createdAt.getTime() - const direction = tx.baseCurrency.type === 'fiat' ? 'buy' : 'sell' + // Map Moonpay status to Edge status + const status: Status = statusMap[tx.status] ?? 'other' + + // Determine direction based on paymentMethod vs payoutMethod + // Buy transactions have paymentMethod, sell transactions have payoutMethod + const direction = tx.paymentMethod != null ? 'buy' : 'sell' + + // Get the payout currency - different field names for buy vs sell + const payoutCurrency = direction === 'buy' ? tx.currency : tx.quoteCurrency + if (payoutCurrency == null) { + throw new Error(`Missing payout currency for tx ${tx.id}`) + } + + // For buy transactions: deposit is fiat (no crypto info), payout is crypto + // For sell transactions: deposit is crypto, payout is fiat (no crypto info) + const depositAsset = + direction === 'sell' + ? processMetadata(tx.baseCurrency.metadata, tx.baseCurrency.code) + : { chainPluginId: undefined, evmChainId: undefined, tokenId: undefined } + + const payoutAsset = + direction === 'buy' + ? processMetadata(payoutCurrency.metadata, payoutCurrency.code) + : { chainPluginId: undefined, evmChainId: undefined, tokenId: undefined } const standardTx: StandardTx = { - status: 'complete', + status, orderId: tx.id, countryCode: tx.country, - depositTxid: direction === 'sell' ? tx.cryptoTransactionId : undefined, + depositTxid: direction === 'sell' ? tx.depositHash : undefined, depositAddress: undefined, depositCurrency: tx.baseCurrency.code.toUpperCase(), + depositChainPluginId: depositAsset.chainPluginId, + depositEvmChainId: depositAsset.evmChainId, + depositTokenId: depositAsset.tokenId, depositAmount: tx.baseCurrencyAmount, direction, exchangeType: 'fiat', paymentType: getFiatPaymentType(tx), payoutTxid: direction === 'buy' ? tx.cryptoTransactionId : undefined, - payoutAddress: tx.walletAddress, - payoutCurrency: tx.currency.code.toUpperCase(), - payoutAmount: tx.quoteCurrencyAmount, + payoutAddress: direction === 'buy' ? tx.walletAddress : undefined, + payoutCurrency: payoutCurrency.code.toUpperCase(), + payoutChainPluginId: payoutAsset.chainPluginId, + payoutEvmChainId: payoutAsset.evmChainId, + payoutTokenId: payoutAsset.tokenId, + payoutAmount: tx.quoteCurrencyAmount ?? 0, timestamp: timestamp / 1000, isoDate, usdValue: -1, @@ -229,74 +333,81 @@ export function processMoonpayTx(rawTx: unknown): StandardTx { return standardTx } -export function processMoonpaySellTx(rawTx: unknown): StandardTx { - const tx: MoonpaySellTx = asMoonpaySellTx(rawTx) - const isoDate = tx.createdAt.toISOString() - const timestamp = tx.createdAt.getTime() - const standardTx: StandardTx = { - status: 'complete', - orderId: tx.id, - - countryCode: tx.country, - depositTxid: tx.depositHash, - depositAddress: undefined, - depositCurrency: tx.baseCurrency.code.toUpperCase(), - depositAmount: tx.baseCurrencyAmount, - direction: 'sell', - exchangeType: 'fiat', - paymentType: getFiatPaymentType(tx), - payoutTxid: undefined, - payoutAddress: undefined, - payoutCurrency: tx.quoteCurrency.code.toUpperCase(), - payoutAmount: tx.quoteCurrencyAmount, - timestamp: timestamp / 1000, - isoDate, - usdValue: -1, - rawTx: rawTx - } - return standardTx +const paymentMethodMap: Record = { + ach_bank_transfer: 'ach', + apple_pay: 'applepay', + credit_debit_card: 'credit', + gbp_bank_transfer: 'fasterpayments', + gbp_open_banking_payment: 'fasterpayments', + google_pay: 'googlepay', + interac: 'interac', + moonpay_balance: 'moonpaybalance', + paypal: 'paypal', + pix_instant_payment: 'pix', + revolut_pay: 'revolut', + sepa_bank_transfer: 'sepa', + venmo: 'venmo', + yellow_card_bank_transfer: 'yellowcard' } -function getFiatPaymentType( - tx: MoonpayTx | MoonpaySellTx -): FiatPaymentType | null { +function getFiatPaymentType(tx: MoonpayTx): FiatPaymentType | null { + let paymentMethod: FiatPaymentType | null = null switch (tx.paymentMethod) { case undefined: return null - case 'ach_bank_transfer': - return 'ach' - case 'apple_pay': - return 'applepay' - case 'credit_debit_card': - return 'credit' - case 'gbp_open_banking_payment': - return 'fasterpayments' - case 'google_pay': - return 'googlepay' case 'mobile_wallet': // Older versions of Moonpay data had a separate cardType field. - return 'cardType' in tx - ? tx.cardType === 'apple_pay' - ? 'applepay' - : tx.cardType === 'google_pay' - ? 'googlepay' - : null - : null - case 'moonpay_balance': - return 'moonpaybalance' - case 'paypal': - return 'paypal' - case 'pix_instant_payment': - return 'pix' - case 'sepa_bank_transfer': - return 'sepa' - case 'venmo': - return 'venmo' - case 'yellow_card_bank_transfer': - return 'yellowcard' + if (tx.cardType === 'apple_pay') { + paymentMethod = 'applepay' + } else if (tx.cardType === 'google_pay') { + paymentMethod = 'googlepay' + } else if (tx.cardType === undefined) { + paymentMethod = 'applepay' + } + break default: - throw new Error( - `Unknown payment method: ${tx.paymentMethod} for ${tx.id}` - ) + paymentMethod = paymentMethodMap[tx.paymentMethod] + break } + if (paymentMethod == null) { + throw new Error(`Unknown payment method: ${tx.paymentMethod} for ${tx.id}`) + } + return paymentMethod +} + +// COMMENT: The reason for using a function over a object map is to encode +// the the `undefined` type which is possible since there is no type safety on +// the input value (key for the mapping) and to also allow for undefined keys +// to always return undefined. + +// Map Moonpay's networkCode to Edge pluginId +const moonpayNetworkToPluginId = ( + moonpayNetwork?: string +): string | undefined => { + if (moonpayNetwork == null) return undefined + return { + algorand: 'algorand', + arbitrum: 'arbitrum', + avalanche_c_chain: 'avalanche', + base: 'base', + binance_smart_chain: 'binancesmartchain', + bitcoin: 'bitcoin', + bitcoin_cash: 'bitcoincash', + cardano: 'cardano', + cosmos: 'cosmoshub', + dogecoin: 'dogecoin', + ethereum: 'ethereum', + ethereum_classic: 'ethereumclassic', + hedera: 'hedera', + litecoin: 'litecoin', + optimism: 'optimism', + polygon: 'polygon', + ripple: 'ripple', + solana: 'solana', + stellar: 'stellar', + sui: 'sui', + ton: 'ton', + tron: 'tron', + zksync: 'zksync' + }[moonpayNetwork] } diff --git a/src/partners/paybis.ts b/src/partners/paybis.ts index 257ee16f..355be8e4 100644 --- a/src/partners/paybis.ts +++ b/src/partners/paybis.ts @@ -22,7 +22,7 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' const PLUGIN_START_DATE = '2023-09-01T00:00:00.000Z' const asStatuses = asMaybe( @@ -155,6 +155,7 @@ const statusMap: { [key in PartnerStatuses]: Status } = { export async function queryPaybis( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asStandardPluginParams(pluginParams) const { apiKey } = apiKeys const nowDate = new Date() @@ -193,7 +194,7 @@ export async function queryPaybis( to: new Date(endTime).toISOString(), limit: QUERY_LIMIT_TXS } - datelog(`Querying from:${queryParams.from} to:${queryParams.to}`) + log(`Querying from:${queryParams.from} to:${queryParams.to}`) if (cursor != null) queryParams.cursor = cursor urlObj.set('query', queryParams) @@ -221,25 +222,23 @@ export async function queryPaybis( if (cursor == null) { break } else { - datelog(`Get nextCursor: ${cursor}`) + log(`Get nextCursor: ${cursor}`) } } const endDate = new Date(endTime) startTime = endTime - datelog( - `Paybis endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}` - ) + log(`endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}`) if (endTime > now) { break } retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${60 * retry}s`) + log.warn(`Snoozing ${60 * retry}s`) await snooze(60000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -282,6 +281,9 @@ export function processPaybisTx(rawTx: unknown): StandardTx { depositTxid, depositAddress: undefined, depositCurrency: spentOriginal.currency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount, direction, exchangeType: 'fiat', @@ -289,6 +291,9 @@ export function processPaybisTx(rawTx: unknown): StandardTx { payoutTxid, payoutAddress: tx.to.address, payoutCurrency: receivedOriginal.currency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount, timestamp, isoDate, diff --git a/src/partners/paytrie.ts b/src/partners/paytrie.ts index 2cb16416..045a87c4 100644 --- a/src/partners/paytrie.ts +++ b/src/partners/paytrie.ts @@ -2,7 +2,6 @@ import { asArray, asNumber, asObject, asString, asUnknown } from 'cleaners' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog } from '../util' const asPaytrieTx = asObject({ inputTXID: asString, @@ -20,6 +19,7 @@ const asPaytrieTxs = asArray(asUnknown) export async function queryPaytrie( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let startDate = '2020-01-01' const endDate = new Date().toISOString().slice(0, 10) @@ -51,7 +51,7 @@ export async function queryPaytrie( method: 'post' } ).catch(err => { - datelog(err) + log.error(String(err)) throw err }) @@ -86,6 +86,9 @@ export function processPaytrieTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: order.inputAddress, depositCurrency: order.inputCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: order.inputAmount, direction: null, // No records of paytrie in the DB to determine exchangeType: 'fiat', // IDK what paytrie is, but I assume it's a fiat exchange @@ -93,6 +96,9 @@ export function processPaytrieTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: order.outputAddress, payoutCurrency: order.outputCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: order.outputAmount, timestamp: new Date(order.timestamp).getTime() / 1000, isoDate: order.timestamp, diff --git a/src/partners/rango.ts b/src/partners/rango.ts new file mode 100644 index 00000000..f8313f68 --- /dev/null +++ b/src/partners/rango.ts @@ -0,0 +1,311 @@ +import { + asArray, + asEither, + asMaybe, + asNull, + asNumber, + asObject, + asOptional, + asString, + asUnknown, + asValue +} from 'cleaners' + +import { + PartnerPlugin, + PluginParams, + PluginResult, + StandardTx, + Status +} from '../types' +import { retryFetch } from '../util' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +// Start date for Rango transactions (first Edge transaction was 2024-06-23) +const RANGO_START_DATE = '2024-06-01T00:00:00.000Z' + +const asRangoPluginParams = asObject({ + settings: asObject({ + latestIsoDate: asOptional(asString, RANGO_START_DATE) + }), + apiKeys: asObject({ + apiKey: asOptional(asString), + secret: asOptional(asString) + }) +}) + +const asRangoStatus = asMaybe( + asValue('success', 'failed', 'running', 'pending'), + 'other' +) + +const asBlockchainData = asObject({ + blockchain: asString, + type: asOptional(asString), + displayName: asOptional(asString) +}) + +const asToken = asObject({ + blockchainData: asBlockchainData, + symbol: asString, + address: asOptional(asEither(asString, asNull)), + decimals: asNumber, + expectedAmount: asOptional(asNumber), + realAmount: asOptional(asNumber) +}) + +const asStepSummary = asObject({ + swapper: asObject({ + swapperId: asString, + swapperTitle: asOptional(asString) + }), + fromToken: asToken, + toToken: asToken, + status: asRangoStatus, + stepNumber: asNumber, + sender: asOptional(asString), + recipient: asOptional(asString), + affiliates: asOptional(asArray(asUnknown)) +}) + +const asRangoTx = asObject({ + requestId: asString, + transactionTime: asString, + status: asRangoStatus, + stepsSummary: asArray(asStepSummary), + feeUsd: asOptional(asNumber), + referrerCode: asOptional(asString) +}) + +const asRangoResult = asObject({ + page: asOptional(asNumber), + offset: asOptional(asNumber), + total: asNumber, + transactions: asArray(asUnknown) +}) + +const PAGE_LIMIT = 20 // API max is 20 per page +const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 3 // 3 days + +type RangoTx = ReturnType +type RangoStatus = ReturnType + +const statusMap: { [key in RangoStatus]: Status } = { + success: 'complete', + failed: 'failed', + running: 'processing', + pending: 'pending', + other: 'other' +} + +// Map Rango blockchain names to Edge pluginIds +const RANGO_BLOCKCHAIN_TO_PLUGIN_ID: Record = { + ARBITRUM: 'arbitrum', + AVAX_CCHAIN: 'avalanche', + BASE: 'base', + BCH: 'bitcoincash', + BINANCE: 'binance', + BSC: 'binancesmartchain', + BTC: 'bitcoin', + CELO: 'celo', + COSMOS: 'cosmoshub', + DOGE: 'dogecoin', + ETH: 'ethereum', + FANTOM: 'fantom', + LTC: 'litecoin', + MATIC: 'polygon', + OPTIMISM: 'optimism', + OSMOSIS: 'osmosis', + POLYGON: 'polygon', + SOLANA: 'solana', + TRON: 'tron', + ZKSYNC: 'zksync' +} + +export async function queryRango( + pluginParams: PluginParams +): Promise { + const { log } = pluginParams + const { settings, apiKeys } = asRangoPluginParams(pluginParams) + const { apiKey, secret } = apiKeys + let { latestIsoDate } = settings + + if (apiKey == null || secret == null) { + return { settings: { latestIsoDate }, transactions: [] } + } + + const standardTxs: StandardTx[] = [] + let startMs = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK + if (startMs < 0) startMs = 0 + + let done = false + let page = 1 + + try { + while (!done) { + // API: https://api-docs.rango.exchange/reference/filtertransactions + // Endpoint: GET https://api.rango.exchange/scanner/tx/filter + // Auth: apiKey and token (secret) in query params + // Date range: start/end in milliseconds + const queryParams = new URLSearchParams({ + apiKey, + token: secret, + limit: String(PAGE_LIMIT), + page: String(page), + order: 'asc', // Oldest to newest + start: String(startMs) + }) + + const request = `https://api.rango.exchange/scanner/tx/filter?${queryParams.toString()}` + + const response = await retryFetch(request, { + method: 'GET', + headers: { 'Content-Type': 'application/json' } + }) + + if (!response.ok) { + const text = await response.text() + throw new Error(`Rango API error ${response.status}: ${text}`) + } + + const json = await response.json() + const result = asRangoResult(json) + + const txs = result.transactions + let processedCount = 0 + + for (const rawTx of txs) { + try { + const standardTx = processRangoTx(rawTx, pluginParams) + standardTxs.push(standardTx) + processedCount++ + + if (standardTx.isoDate > latestIsoDate) { + latestIsoDate = standardTx.isoDate + } + } catch (e) { + // Log but continue processing other transactions + log.warn(`Failed to process tx: ${String(e)}`) + } + } + + const currentOffset = (page - 1) * PAGE_LIMIT + txs.length + log( + `Page ${page} (offset ${currentOffset}/${result.total}): processed ${processedCount}, latestIsoDate ${latestIsoDate}` + ) + + page++ + + // Reached end of results + if (txs.length < PAGE_LIMIT || currentOffset >= result.total) { + done = true + } + } + } catch (e) { + log.error(String(e)) + // Do not throw - save progress since we query from oldest to newest + // This ensures we don't lose transactions on transient failures + } + + const out: PluginResult = { + settings: { latestIsoDate }, + transactions: standardTxs + } + return out +} + +export const rango: PartnerPlugin = { + queryFunc: queryRango, + pluginName: 'Rango', + pluginId: 'rango' +} + +export function processRangoTx( + rawTx: unknown, + pluginParams: PluginParams +): StandardTx { + const { log } = pluginParams + const tx: RangoTx = asRangoTx(rawTx) + + // Parse the ISO date string (e.g., "2025-12-24T15:43:46.926+00:00") + const date = new Date(tx.transactionTime) + const timestamp = Math.floor(date.getTime() / 1000) + const isoDate = date.toISOString() + + // Get first and last steps for deposit/payout info + const firstStep = tx.stepsSummary[0] + const lastStep = tx.stepsSummary[tx.stepsSummary.length - 1] + + if (firstStep == null || lastStep == null) { + throw new Error(`Transaction ${tx.requestId} has no steps`) + } + + // Deposit info from first step + const depositBlockchain = firstStep.fromToken.blockchainData.blockchain + const depositChainPluginId = RANGO_BLOCKCHAIN_TO_PLUGIN_ID[depositBlockchain] + if (depositChainPluginId == null) { + throw new Error( + `Unknown Rango blockchain "${depositBlockchain}". Add mapping to RANGO_BLOCKCHAIN_TO_PLUGIN_ID.` + ) + } + const depositEvmChainId = EVM_CHAIN_IDS[depositChainPluginId] + + // Payout info from last step + const payoutBlockchain = lastStep.toToken.blockchainData.blockchain + const payoutChainPluginId = RANGO_BLOCKCHAIN_TO_PLUGIN_ID[payoutBlockchain] + if (payoutChainPluginId == null) { + throw new Error( + `Unknown Rango blockchain "${payoutBlockchain}". Add mapping to RANGO_BLOCKCHAIN_TO_PLUGIN_ID.` + ) + } + const payoutEvmChainId = EVM_CHAIN_IDS[payoutChainPluginId] + + // Get amounts - prefer realAmount, fall back to expectedAmount + const depositAmount = + firstStep.fromToken.realAmount ?? firstStep.fromToken.expectedAmount ?? 0 + const payoutAmount = + lastStep.toToken.realAmount ?? lastStep.toToken.expectedAmount ?? 0 + + const dateStr = isoDate.split('T')[0] + const depositCurrency = firstStep.fromToken.symbol + const depositTokenId = firstStep.fromToken.address ?? null + const payoutCurrency = lastStep.toToken.symbol + const payoutTokenId = lastStep.toToken.address ?? null + + log( + `${dateStr} ${depositCurrency} ${depositAmount} ${depositChainPluginId}${ + depositTokenId != null ? ` ${depositTokenId}` : '' + } -> ${payoutCurrency} ${payoutAmount} ${payoutChainPluginId}${ + payoutTokenId != null ? ` ${payoutTokenId}` : '' + }` + ) + + const standardTx: StandardTx = { + status: statusMap[tx.status], + orderId: tx.requestId, + countryCode: null, + depositTxid: undefined, + depositAddress: firstStep.sender, + depositCurrency: firstStep.fromToken.symbol, + depositChainPluginId, + depositEvmChainId, + depositTokenId, + depositAmount, + direction: null, + exchangeType: 'swap', + paymentType: null, + payoutTxid: undefined, + payoutAddress: lastStep.recipient, + payoutCurrency: lastStep.toToken.symbol, + payoutChainPluginId, + payoutEvmChainId, + payoutTokenId: lastStep.toToken.address ?? null, + payoutAmount, + timestamp, + isoDate, + usdValue: -1, + rawTx + } + + return standardTx +} diff --git a/src/partners/safello.ts b/src/partners/safello.ts index 07f3a399..1dbcca17 100644 --- a/src/partners/safello.ts +++ b/src/partners/safello.ts @@ -92,6 +92,9 @@ export function processSafelloTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: tx.currency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.amount, direction: 'buy', exchangeType: 'fiat', @@ -99,6 +102,9 @@ export function processSafelloTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: tx.cryptoCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: 0, timestamp: timestamp / 1000, isoDate: date.toISOString(), diff --git a/src/partners/shapeshift.ts b/src/partners/shapeshift.ts index af05144b..126fdb96 100644 --- a/src/partners/shapeshift.ts +++ b/src/partners/shapeshift.ts @@ -2,7 +2,7 @@ import { asArray, asNumber, asObject, asString, asUnknown } from 'cleaners' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, safeParseFloat } from '../util' +import { safeParseFloat } from '../util' const asShapeshiftTx = asObject({ orderId: asString, @@ -25,6 +25,7 @@ const asShapeshiftResult = asArray(asUnknown) export async function queryShapeshift( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let apiKey @@ -60,7 +61,7 @@ export async function queryShapeshift( // done = true // } } catch (e) { - datelog(e) + log.error(String(e)) throw e } // page++ @@ -92,6 +93,9 @@ export function processShapeshiftTx(rawTx: unknown): StandardTx { depositTxid: tx.inputTXID, depositAddress: tx.inputAddress, depositCurrency: tx.inputCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.inputAmount, direction: null, exchangeType: 'swap', @@ -99,6 +103,9 @@ export function processShapeshiftTx(rawTx: unknown): StandardTx { payoutTxid: tx.outputTXID, payoutAddress: tx.outputAddress, payoutCurrency: tx.outputCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat(tx.outputAmount), timestamp: tx.timestamp, isoDate: new Date(tx.timestamp * 1000).toISOString(), diff --git a/src/partners/sideshift.ts b/src/partners/sideshift.ts index e6b07efe..303973c3 100644 --- a/src/partners/sideshift.ts +++ b/src/partners/sideshift.ts @@ -16,7 +16,115 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { + ChainNameToPluginIdMapping, + createTokenId, + EdgeTokenId, + tokenTypes +} from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +// Map Sideshift network names to Edge pluginId +const SIDESHIFT_NETWORK_TO_PLUGIN_ID: ChainNameToPluginIdMapping = { + algorand: 'algorand', + arbitrum: 'arbitrum', + avax: 'avalanche', + base: 'base', + bitcoin: 'bitcoin', + bitcoincash: 'bitcoincash', + bsc: 'binancesmartchain', + cardano: 'cardano', + cosmos: 'cosmoshub', + dash: 'dash', + doge: 'dogecoin', + ethereum: 'ethereum', + fantom: 'fantom', + litecoin: 'litecoin', + monero: 'monero', + optimism: 'optimism', + polkadot: 'polkadot', + polygon: 'polygon', + ripple: 'ripple', + rootstock: 'rsk', + solana: 'solana', + sonic: 'sonic', + stellar: 'stellar', + sui: 'sui', + ton: 'ton', + tron: 'tron', + xec: 'ecash', + zcash: 'zcash', + zksyncera: 'zksync' +} + +// Some assets have different names in the API vs transaction data +// Map: `${txAsset}-${network}` -> API coin name +const ASSET_NAME_OVERRIDES: Record = { + 'USDT-arbitrum': 'USDT0', + 'USDT-polygon': 'USDT0', + 'USDT-hyperevm': 'USDT0' +} + +// Delisted coins that are no longer in the SideShift API +// Map: `${coin}-${network}` -> contract address (null for native gas tokens) +const DELISTED_COINS: Record = { + 'BUSD-bsc': '0xe9e7CEA3DedcA5984780Bafc599bD69ADd087D56', + 'FTM-fantom': null, // Native gas token + 'MATIC-ethereum': '0x7d1afa7b718fb893db30a3abc0cfc608aacfebb0', + 'MATIC-polygon': null, // Native gas token (rebranded to POL) + 'MKR-ethereum': '0x9f8F72aA9304c8B593d555F12eF6589cC3A579A2', + 'PYTH-solana': 'HZ1JovNiVvGrGNiiYvEozEVgZ58xaU3RKwX8eACQBCt3', + 'USDC-tron': 'TEkxiTehnzSmSe2XqrBj4w32RUN966rdz8', + 'XMR-monero': null, // Native gas token + 'ZEC-zcash': null // Native gas token +} + +// Cleaners for Sideshift coins API response +const asSideshiftTokenDetails = asObject({ + contractAddress: asString +}) + +const asSideshiftCoin = asObject({ + coin: asString, + networks: asArray(asString), + tokenDetails: asOptional( + asObject((raw: unknown) => asSideshiftTokenDetails(raw)) + ) +}) + +const asSideshiftCoinsResponse = asArray(asSideshiftCoin) + +// Cache for Sideshift coins data +// Key: `${coin}-${network}` -> contract address or null for mainnet coins +let sideshiftCoinsCache: Map | null = null + +async function fetchSideshiftCoins(): Promise> { + if (sideshiftCoinsCache != null) { + return sideshiftCoinsCache + } + + const cache = new Map() + + const response = await retryFetch('https://sideshift.ai/api/v2/coins') + if (!response.ok) { + throw new Error(`Failed to fetch sideshift coins: ${response.status}`) + } + + const coins = asSideshiftCoinsResponse(await response.json()) + + for (const coin of coins) { + for (const network of coin.networks) { + const key = `${coin.coin.toUpperCase()}-${network}` + // Get contract address from tokenDetails if available + const tokenDetail = coin.tokenDetails?.[network] + cache.set(key, tokenDetail?.contractAddress ?? null) + } + } + + sideshiftCoinsCache = cache + return cache +} const asSideshiftStatus = asMaybe( asValue( @@ -40,14 +148,14 @@ const asSideshiftTx = asObject({ depositAddress: asMaybe(asObject({ address: asMaybe(asString) })), prevDepositAddresses: asMaybe(asObject({ address: asMaybe(asString) })), depositAsset: asString, - // depositMethodId: asString, + depositNetwork: asOptional(asString), invoiceAmount: asString, settleAddress: asObject({ address: asString }), - // settleMethodId: asString, settleAmount: asString, settleAsset: asString, + settleNetwork: asOptional(asString), createdAt: asString }) @@ -97,6 +205,7 @@ function affiliateSignature( export async function querySideshift( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const { settings, apiKeys } = asSideshiftPluginParams(pluginParams) const { sideshiftAffiliateId, sideshiftAffiliateSecret } = apiKeys let { latestIsoDate } = settings @@ -131,24 +240,24 @@ export async function querySideshift( break } for (const rawTx of orders) { - const standardTx = processSideshiftTx(rawTx) + const standardTx = await processSideshiftTx(rawTx, pluginParams) standardTxs.push(standardTx) if (standardTx.isoDate > latestIsoDate) { latestIsoDate = standardTx.isoDate } } startTime = new Date(latestIsoDate).getTime() - datelog(`Sideshift latestIsoDate ${latestIsoDate}`) + log(`latestIsoDate ${latestIsoDate}`) if (endTime > now) { break } retry = 0 } catch (e) { - datelog(e) + log.error(String(e)) // Retry a few times with time delay to prevent throttling retry++ if (retry <= MAX_RETRIES) { - datelog(`Snoozing ${5 * retry}s`) + log.warn(`Snoozing ${5 * retry}s`) await snooze(5000 * retry) } else { // We can safely save our progress since we go from oldest to newest. @@ -170,12 +279,79 @@ export const sideshift: PartnerPlugin = { pluginId: 'sideshift' } -export function processSideshiftTx(rawTx: unknown): StandardTx { +interface EdgeAssetInfo { + chainPluginId: string | undefined + evmChainId: number | undefined + tokenId: EdgeTokenId +} + +/** + * Process network and asset info to extract Edge asset info + */ +async function getAssetInfo( + network: string | undefined, + asset: string +): Promise { + if (network == null) { + throw new Error(`Missing network for asset: ${asset}`) + } + + const chainPluginId = SIDESHIFT_NETWORK_TO_PLUGIN_ID[network] + if (chainPluginId == null) { + throw new Error(`Unknown network: ${network}`) + } + + // Get evmChainId if this is an EVM chain + const evmChainId = EVM_CHAIN_IDS[chainPluginId] + + // Get contract address from cache + const coinsCache = await fetchSideshiftCoins() + + // Check for asset name overrides (e.g., USDT -> USDT0 on certain networks) + const overrideKey = `${asset.toUpperCase()}-${network}` + const apiCoinName = ASSET_NAME_OVERRIDES[overrideKey] ?? asset.toUpperCase() + const cacheKey = `${apiCoinName}-${network}` + + // Check cache first, then fall back to delisted coins mapping + let contractAddress: string | null | undefined + if (coinsCache.has(cacheKey)) { + contractAddress = coinsCache.get(cacheKey) + } else if (overrideKey in DELISTED_COINS) { + contractAddress = DELISTED_COINS[overrideKey] + } else { + throw new Error(`Unknown coin: ${asset} on network ${network}`) + } + + // Determine tokenId + // contractAddress === null means mainnet coin (tokenId = null) + // contractAddress === string means token (tokenId = createTokenId(...)) + let tokenId: EdgeTokenId = null + if (contractAddress != null) { + const tokenType = tokenTypes[chainPluginId] + if (tokenType == null) { + throw new Error( + `Unknown tokenType for chainPluginId ${chainPluginId} (asset: ${asset})` + ) + } + tokenId = createTokenId(tokenType, asset.toUpperCase(), contractAddress) + } + + return { chainPluginId, evmChainId, tokenId } +} + +export async function processSideshiftTx( + rawTx: unknown, + pluginParams: PluginParams +): Promise { const tx: SideshiftTx = asSideshiftTx(rawTx) const depositAddress = tx.depositAddress?.address ?? tx.prevDepositAddresses?.address const { isoDate, timestamp } = smartIsoDateFromTimestamp(tx.createdAt) + // Get asset info for deposit and payout + const depositAsset = await getAssetInfo(tx.depositNetwork, tx.depositAsset) + const payoutAsset = await getAssetInfo(tx.settleNetwork, tx.settleAsset) + const standardTx: StandardTx = { status: statusMap[tx.status], orderId: tx.id, @@ -183,6 +359,9 @@ export function processSideshiftTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress, depositCurrency: tx.depositAsset, + depositChainPluginId: depositAsset.chainPluginId, + depositEvmChainId: depositAsset.evmChainId, + depositTokenId: depositAsset.tokenId, depositAmount: Number(tx.invoiceAmount), direction: null, exchangeType: 'swap', @@ -190,6 +369,9 @@ export function processSideshiftTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: tx.settleAddress.address, payoutCurrency: tx.settleAsset, + payoutChainPluginId: payoutAsset.chainPluginId, + payoutEvmChainId: payoutAsset.evmChainId, + payoutTokenId: payoutAsset.tokenId, payoutAmount: Number(tx.settleAmount), timestamp, isoDate, diff --git a/src/partners/simplex.ts b/src/partners/simplex.ts index ebc8de61..40f5f62a 100644 --- a/src/partners/simplex.ts +++ b/src/partners/simplex.ts @@ -12,7 +12,6 @@ import { import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' import { safeParseFloat } from '../util' -import { isFiatCurrency } from '../util/fiatCurrency' const asSimplexTx = asObject({ amount_usd: asString, @@ -150,6 +149,9 @@ export function processSimplexTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: tx.currency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: safeParseFloat(tx.fiat_total_amount), direction: 'buy', exchangeType: 'fiat', @@ -157,6 +159,9 @@ export function processSimplexTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: tx.crypto_currency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat(tx.amount_crypto), timestamp: tx.created_at, isoDate: new Date(tx.created_at * 1000).toISOString(), diff --git a/src/partners/swapuz.ts b/src/partners/swapuz.ts index e187c993..68d146b3 100644 --- a/src/partners/swapuz.ts +++ b/src/partners/swapuz.ts @@ -15,8 +15,7 @@ import { StandardTx, Status } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp } from '../util' -import { isFiatCurrency } from '../util/fiatCurrency' +import { retryFetch, smartIsoDateFromTimestamp } from '../util' const asSwapuzLogin = asObject({ result: asObject({ @@ -62,6 +61,7 @@ const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days export const querySwapuz = async ( pluginParams: PluginParams ): Promise => { + const { log } = pluginParams const standardTxs: StandardTx[] = [] const { settings, apiKeys } = asSwapuzPluginParams(pluginParams) @@ -127,20 +127,18 @@ export const querySwapuz = async ( oldestIsoDate = standardTx.isoDate } if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `Swapuz done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` - ) + log(`Done: date ${standardTx.isoDate} < ${previousLatestIsoDate}`) done = true } } - datelog(`Swapuz page=${page}/${maxPage} oldestIsoDate: ${oldestIsoDate}`) + log(`page=${page}/${maxPage} oldestIsoDate: ${oldestIsoDate}`) if (currentPage >= maxPage) { break } } catch (e) { const err: any = e - datelog(err.message) + log.error(err.message) throw e } } @@ -177,6 +175,9 @@ export function processSwapuzTx(rawTx: unknown): StandardTx { depositTxid: tx.dTxId ?? tx.depositTransactionID, depositCurrency: tx.from.toUpperCase(), depositAddress: tx.depositAddress, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.amount, direction: null, exchangeType: 'swap', @@ -184,6 +185,9 @@ export function processSwapuzTx(rawTx: unknown): StandardTx { payoutTxid: tx.wTxId ?? tx.withdrawalTransactionID, payoutCurrency: tx.to.toUpperCase(), payoutAddress: undefined, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.amountResult, timestamp, isoDate, diff --git a/src/partners/switchain.ts b/src/partners/switchain.ts index 83844b90..6cb01859 100644 --- a/src/partners/switchain.ts +++ b/src/partners/switchain.ts @@ -2,7 +2,7 @@ import { asArray, asObject, asString, asUnknown } from 'cleaners' import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, safeParseFloat } from '../util' +import { safeParseFloat } from '../util' import { queryDummy } from './dummy' const asSwitchainTx = asObject({ @@ -32,6 +32,7 @@ const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 4 // 4 days ago export async function querySwitchain( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let apiKey let latestTimestamp = 0 @@ -66,7 +67,7 @@ export async function querySwitchain( result = asSwitchainResult(await response.json()) } } catch (e) { - datelog(e) + log.error(String(e)) throw e } @@ -120,6 +121,9 @@ export function processSwitchainTx(rawTx: unknown): StandardTx { depositTxid: tx.depositTxId, depositAddress: tx.depositAddress, depositCurrency: pair[0].toUpperCase(), + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: safeParseFloat(tx.amountFrom), direction: null, exchangeType: 'swap', @@ -127,6 +131,9 @@ export function processSwitchainTx(rawTx: unknown): StandardTx { payoutTxid: tx.withdrawTxId, payoutAddress: tx.withdrawAddress, payoutCurrency: pair[1].toUpperCase(), + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat(tx.rate), timestamp: timestamp / 1000, isoDate: tx.createdAt, diff --git a/src/partners/thorchain.ts b/src/partners/thorchain.ts index 6738de33..edf123dc 100644 --- a/src/partners/thorchain.ts +++ b/src/partners/thorchain.ts @@ -11,7 +11,79 @@ import { import { HeadersInit } from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' +import { + ChainNameToPluginIdMapping, + createTokenId, + EdgeTokenId, + tokenTypes +} from '../util/asEdgeTokenId' +import { EVM_CHAIN_IDS } from '../util/chainIds' + +// Thorchain chain names to Edge pluginIds +const THORCHAIN_CHAIN_TO_PLUGINID: ChainNameToPluginIdMapping = { + ARB: 'arbitrum', + BASE: 'base', + BTC: 'bitcoin', + DASH: 'dash', + ETH: 'ethereum', + LTC: 'litecoin', + DOGE: 'dogecoin', + XRP: 'ripple', + BCH: 'bitcoincash', + BSC: 'binancesmartchain', + BNB: 'binancesmartchain', + AVAX: 'avalanche', + TRON: 'tron', + THOR: 'thorchainrune', + GAIA: 'cosmoshub', + KUJI: 'kujira', + MAYA: 'mayachain', + ZEC: 'zcash' +} + +interface ParsedThorchainAsset { + chain: string + asset: string + contractAddress?: string +} + +/** + * Parse Thorchain asset string format: "CHAIN.ASSET" or "CHAIN.ASSET-CONTRACT" + * Examples: + * "BTC.BTC" -> { chain: "BTC", asset: "BTC" } + * "ETH.USDT-0XDAC17F958D2EE523A2206206994597C13D831EC7" -> { chain: "ETH", asset: "USDT", contractAddress: "0XDAC..." } + */ +function parseThorchainAsset(assetString: string): ParsedThorchainAsset { + const [chainAssetPart, contractAddress] = assetString.split('-') + const [chain, asset] = chainAssetPart.split('.') + return { chain, asset, contractAddress } +} + +/** + * Get Edge asset info (pluginId, evmChainId, tokenId) from Thorchain asset string + */ +function getEdgeAssetInfo( + assetString: string +): { + asset: string + pluginId: string + evmChainId: number | undefined + tokenId: EdgeTokenId +} { + const { chain, asset, contractAddress } = parseThorchainAsset(assetString) + + const pluginId = THORCHAIN_CHAIN_TO_PLUGINID[chain] + if (pluginId == null) { + throw new Error(`Unknown Thorchain chain: ${chain}`) + } + + const evmChainId = EVM_CHAIN_IDS[pluginId] + const tokenType = tokenTypes[pluginId] + const tokenId = createTokenId(tokenType, asset, contractAddress) + + return { asset, pluginId, evmChainId, tokenId } +} const asThorchainTx = asObject({ date: asString, @@ -91,6 +163,7 @@ const makeThorchainPlugin = (info: ThorchainInfo): PartnerPlugin => { const queryThorchain = async ( pluginParams: PluginParams ): Promise => { + const { log } = pluginParams const standardTxs: StandardTx[] = [] const pluginParamsClean = asThorchainPluginParams(pluginParams) @@ -98,6 +171,8 @@ const makeThorchainPlugin = (info: ThorchainInfo): PartnerPlugin => { const { affiliateAddress, thorchainAddress, xClientId } = apiKeys let { latestIsoDate } = settings + const processTx = makeThorchainProcessTx(info) + let previousTimestamp = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK if (previousTimestamp < 0) previousTimestamp = 0 const previousLatestIsoDate = new Date(previousTimestamp).toISOString() @@ -130,55 +205,59 @@ const makeThorchainPlugin = (info: ThorchainInfo): PartnerPlugin => { const resultJson = await result.json() jsonObj = asThorchainResult(resultJson) } catch (e) { - datelog(e) + log.error(String(e)) throw e } const txs = jsonObj.actions for (const rawTx of txs) { - const standardTx = processThorchainTx(rawTx, info, pluginParamsClean) + try { + const standardTx = processTx(rawTx, pluginParams) - // Handle null case as a continue - if (standardTx == null) { - continue - } + // Handle null case as a continue + if (standardTx == null) { + continue + } - // See if the transaction exists already - const previousTxIndex = standardTxs.findIndex( - tx => - tx.orderId === standardTx.orderId && - tx.timestamp === standardTx.timestamp && - tx.depositCurrency === standardTx.depositCurrency && - tx.payoutCurrency === standardTx.payoutCurrency && - tx.payoutAmount === standardTx.payoutAmount && - tx.depositAmount !== standardTx.depositAmount - ) - if (previousTxIndex === -1) { - standardTxs.push(standardTx) - } else { - const previousTx = standardTxs[previousTxIndex] - const previousRawTxs: unknown[] = Array.isArray(previousTx.rawTx) - ? previousTx.rawTx - : [previousTx.rawTx] - const updatedStandardTx = processThorchainTx( - [...previousRawTxs, standardTx.rawTx], - info, - pluginParamsClean + // See if the transaction exists already + const previousTxIndex = standardTxs.findIndex( + tx => + tx.orderId === standardTx.orderId && + tx.timestamp === standardTx.timestamp && + tx.depositCurrency === standardTx.depositCurrency && + tx.payoutCurrency === standardTx.payoutCurrency && + tx.payoutAmount === standardTx.payoutAmount && + tx.depositAmount !== standardTx.depositAmount ) - if (updatedStandardTx != null) { - standardTxs.splice(previousTxIndex, 1, updatedStandardTx) + if (previousTxIndex === -1) { + standardTxs.push(standardTx) + } else { + const previousTx = standardTxs[previousTxIndex] + const previousRawTxs: unknown[] = Array.isArray(previousTx.rawTx) + ? previousTx.rawTx + : [previousTx.rawTx] + const updatedStandardTx = processTx( + [...previousRawTxs, standardTx.rawTx], + pluginParams + ) + if (updatedStandardTx != null) { + standardTxs.splice(previousTxIndex, 1, updatedStandardTx) + } } - } - if (standardTx.isoDate > latestIsoDate) { - latestIsoDate = standardTx.isoDate - } - if (standardTx.isoDate < oldestIsoDate) { - oldestIsoDate = standardTx.isoDate - } - if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `Thorchain done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` + if (standardTx.isoDate > latestIsoDate) { + latestIsoDate = standardTx.isoDate + } + if (standardTx.isoDate < oldestIsoDate) { + oldestIsoDate = standardTx.isoDate + } + if (standardTx.isoDate < previousLatestIsoDate && !done) { + log(`done: date ${standardTx.isoDate} < ${previousLatestIsoDate}`) + done = true + } + } catch (e) { + log.error( + `Error processing tx ${JSON.stringify(rawTx, null, 2)}: ${e}` ) - done = true + throw e } } @@ -201,137 +280,160 @@ const makeThorchainPlugin = (info: ThorchainInfo): PartnerPlugin => { } } -export const thorchain = makeThorchainPlugin({ +export const THORCHAIN_INFO: ThorchainInfo = { pluginName: 'Thorchain', pluginId: 'thorchain', midgardUrl: 'midgard.ninerealms.com' -}) +} -export const maya = makeThorchainPlugin({ +export const MAYA_INFO: ThorchainInfo = { pluginName: 'Maya', pluginId: 'maya', midgardUrl: 'midgard.mayachain.info' -}) +} -export function processThorchainTx( - rawTx: unknown, - info: ThorchainInfo, - pluginParams: ThorchainPluginParams -): StandardTx | null { +export function makeThorchainProcessTx( + info: ThorchainInfo +): (rawTx: unknown, pluginParams?: PluginParams) => StandardTx | null { const { pluginId } = info - const { affiliateAddress, thorchainAddress } = pluginParams.apiKeys - const rawTxs: unknown[] = Array.isArray(rawTx) ? rawTx : [rawTx] - const txs = asArray(asThorchainTx)(rawTxs) - const tx = txs[0] + return (rawTx: unknown, pluginParams?: PluginParams): StandardTx | null => { + if (pluginParams == null) { + throw new Error(`${pluginId}: Missing pluginParams`) + } + const { affiliateAddress, thorchainAddress } = asThorchainPluginParams( + pluginParams + ).apiKeys + const rawTxs: unknown[] = Array.isArray(rawTx) ? rawTx : [rawTx] + const txs = asArray(asThorchainTx)(rawTxs) + const tx = txs[0] + + if (tx == null) { + throw new Error(`${pluginId}: Missing rawTx`) + } - if (tx == null) { - throw new Error(`${pluginId}: Missing rawTx`) - } + const { swap } = tx.metadata + if (swap?.affiliateAddress !== affiliateAddress) { + return null + } - const { swap } = tx.metadata - if (swap?.affiliateAddress !== affiliateAddress) { - return null - } + if (tx.status !== 'success') { + return null + } - if (tx.status !== 'success') { - return null - } + // There must be an affiliate output + const affiliateOut = tx.out.some( + o => o.affiliate === true || o.address === thorchainAddress + ) + if (!affiliateOut) { + return null + } - // There must be an affiliate output - const affiliateOut = tx.out.some( - o => o.affiliate === true || o.address === thorchainAddress - ) - if (!affiliateOut) { - return null - } + // Find the source asset + if (tx.in.length !== 1) { + throw new Error( + `${pluginId}: Unexpected ${tx.in.length} txIns. Expected 1` + ) + } + const txIn = tx.in[0] + if (txIn.coins.length !== 1) { + throw new Error( + `${pluginId}: Unexpected ${txIn.coins.length} txIn.coins. Expected 1` + ) + } + const depositAmount = txs.reduce((sum, txInternal) => { + const amount = + Number(txInternal.in[0].coins[0].amount) / THORCHAIN_MULTIPLIER + return sum + amount + }, 0) + + const srcDestMatch = tx.out.some(o => { + const match = o.coins.some( + c => c.asset === txIn.coins[0].asset && o.affiliate !== true + ) + return match + }) - // Find the source asset - if (tx.in.length !== 1) { - throw new Error(`${pluginId}: Unexpected ${tx.in.length} txIns. Expected 1`) - } - const txIn = tx.in[0] - if (txIn.coins.length !== 1) { - throw new Error( - `${pluginId}: Unexpected ${txIn.coins.length} txIn.coins. Expected 1` - ) - } - const depositAmount = txs.reduce((sum, txInternal) => { - const amount = - Number(txInternal.in[0].coins[0].amount) / THORCHAIN_MULTIPLIER - return sum + amount - }, 0) - - const srcDestMatch = tx.out.some(o => { - const match = o.coins.some( - c => c.asset === txIn.coins[0].asset && o.affiliate !== true + // If there is a match between source and dest asset that means a refund was made + // and the transaction failed + if (srcDestMatch) { + return null + } + + const timestampMs = div(tx.date, '1000000', 16) + const { timestamp, isoDate } = smartIsoDateFromTimestamp( + Number(timestampMs) ) - return match - }) - // If there is a match between source and dest asset that means a refund was made - // and the transaction failed - if (srcDestMatch) { - return null - } + // Parse deposit asset info + const depositAssetString = txIn.coins[0].asset + const depositAssetInfo = getEdgeAssetInfo(depositAssetString) + + // Find the first output that does not match the affiliate address + // as this is assumed to be the true destination asset/address + // If we can't find one, then just match the affiliate address as + // this means the affiliate address is the actual destination. + const hasAffiliateFlag = tx.out.some(o => o.affiliate === true) + let txOut = tx.out.find(out => { + if (hasAffiliateFlag) { + return out.affiliate !== true + } else { + return out.address !== thorchainAddress + } + }) - const timestampMs = div(tx.date, '1000000', 16) - const { timestamp, isoDate } = smartIsoDateFromTimestamp(Number(timestampMs)) - - const [chainAsset] = txIn.coins[0].asset.split('-') - const [, asset] = chainAsset.split('.') - - // Find the first output that does not match the affiliate address - // as this is assumed to be the true destination asset/address - // If we can't find one, then just match the affiliate address as - // this means the affiliate address is the actual destination. - const hasAffiliateFlag = tx.out.some(o => o.affiliate === true) - let txOut = tx.out.find(out => { - if (hasAffiliateFlag) { - return out.affiliate !== true - } else { - return out.address !== thorchainAddress + if (txOut == null) { + // If there are two pools but only one output, there's a problem and we should skip + // this transaction. Midgard sometimes doesn't return the correct output until the transaction + // has completed for awhile. + if (tx.pools.length === 2 && tx.out.length === 1) { + return null + } else if (tx.pools.length === 1 && tx.out.length === 1) { + // The output is a native currency output (maya/rune) + txOut = tx.out[0] + } else { + throw new Error(`${pluginId}: Cannot find output`) + } } - }) - if (txOut == null) { - // If there are two pools but only one output, there's a problem and we should skip - // this transaction. Midgard sometimes doesn't return the correct output until the transaction - // has completed for awhile. - if (tx.pools.length === 2 && tx.out.length === 1) { - return null - } else if (tx.pools.length === 1 && tx.out.length === 1) { - // The output is a native currency output (maya/rune) - txOut = tx.out[0] - } else { - throw new Error(`${pluginId}: Cannot find output`) + // Parse payout asset info + const payoutAssetString = txOut.coins[0].asset + const payoutAssetInfo = getEdgeAssetInfo(payoutAssetString) + + const payoutCurrency = payoutAssetInfo.asset + const payoutAmount = Number(txOut.coins[0].amount) / THORCHAIN_MULTIPLIER + + const standardTx: StandardTx = { + status: 'complete', + orderId: tx.in[0].txID, + countryCode: null, + depositTxid: tx.in[0].txID, + depositAddress: undefined, + depositCurrency: depositAssetInfo.asset.toUpperCase(), + depositChainPluginId: depositAssetInfo.pluginId, + depositEvmChainId: depositAssetInfo.evmChainId, + depositTokenId: depositAssetInfo.tokenId, + depositAmount, + direction: null, + exchangeType: 'swap', + paymentType: null, + payoutTxid: txOut.txID, + payoutAddress: txOut.address, + payoutCurrency, + payoutChainPluginId: payoutAssetInfo.pluginId, + payoutEvmChainId: payoutAssetInfo.evmChainId, + payoutTokenId: payoutAssetInfo.tokenId, + payoutAmount, + timestamp, + isoDate, + usdValue: -1, + rawTx } + return standardTx } - - const [destChainAsset] = txOut.coins[0].asset.split('-') - const [, destAsset] = destChainAsset.split('.') - const payoutCurrency = destAsset - const payoutAmount = Number(txOut.coins[0].amount) / THORCHAIN_MULTIPLIER - - const standardTx: StandardTx = { - status: 'complete', - orderId: tx.in[0].txID, - countryCode: null, - depositTxid: tx.in[0].txID, - depositAddress: undefined, - depositCurrency: asset.toUpperCase(), - depositAmount, - direction: null, - exchangeType: 'swap', - paymentType: null, - payoutTxid: txOut.txID, - payoutAddress: txOut.address, - payoutCurrency, - payoutAmount, - timestamp, - isoDate, - usdValue: -1, - rawTx - } - return standardTx } + +export const thorchain = makeThorchainPlugin(THORCHAIN_INFO) +export const maya = makeThorchainPlugin(MAYA_INFO) +export const processThorchainTx = makeThorchainProcessTx(THORCHAIN_INFO) +export const processMayaTx = makeThorchainProcessTx(MAYA_INFO) diff --git a/src/partners/totle.ts b/src/partners/totle.ts index 746c056c..32a975ea 100644 --- a/src/partners/totle.ts +++ b/src/partners/totle.ts @@ -4,7 +4,7 @@ import fetch from 'node-fetch' import Web3 from 'web3' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, safeParseFloat } from '../util' +import { safeParseFloat } from '../util' import { queryDummy } from './dummy' const asCurrentBlockResult = asNumber @@ -299,6 +299,7 @@ const PRIMARY_ABI: any = [ export async function queryTotle( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const nodeEndpoint = pluginParams.apiKeys.nodeEndpoint // Grab node endpoint from 'reports_apps' database const web3 = new Web3(nodeEndpoint) // Create new Web3 instance using node endpoint const ssFormatTxs: StandardTx[] = [] @@ -395,6 +396,9 @@ export async function queryTotle( depositTxid: receipt.transactionHash, depositAddress: receipt.from, depositCurrency: sourceToken.symbol, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: safeParseFloat( div( sourceAmount.toString(), @@ -409,6 +413,9 @@ export async function queryTotle( payoutTxid: receipt.transactionHash, payoutAddress: receipt.to, payoutCurrency: destinationToken.symbol, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat( div( destinationAmount.toString(), @@ -423,12 +430,12 @@ export async function queryTotle( rawTx: rawSwapEvent } ssFormatTxs.push(ssTx) - datelog(`TOTLE: Currently saved ${ssFormatTxs.length} transactions.`) + log(`Currently saved ${ssFormatTxs.length} transactions.`) } } } } catch (err) { - datelog(err) + log.error(String(err)) } const out: PluginResult = { diff --git a/src/partners/transak.ts b/src/partners/transak.ts index efdb31eb..bd0f6390 100644 --- a/src/partners/transak.ts +++ b/src/partners/transak.ts @@ -17,7 +17,6 @@ import { PluginResult, StandardTx } from '../types' -import { datelog } from '../util' const PAGE_LIMIT = 100 const OFFSET_ROLLBACK = 500 @@ -49,6 +48,7 @@ const asTransakResult = asObject({ export async function queryTransak( pluginParams: PluginParams ): Promise { + const { log } = pluginParams const standardTxs: StandardTx[] = [] let apiKey: string @@ -71,7 +71,7 @@ export async function queryTransak( const result = await fetch(url) resultJSON = asTransakResult(await result.json()) } catch (e) { - datelog(e) + log.error(String(e)) break } const txs = resultJSON.response @@ -129,6 +129,9 @@ export function processTransakTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress, depositCurrency: tx.fiatCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.fiatAmount, direction, exchangeType: 'fiat', @@ -136,6 +139,9 @@ export function processTransakTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: tx.walletAddress, payoutCurrency: tx.cryptoCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.cryptoAmount, timestamp: date.getTime() / 1000, isoDate: date.toISOString(), diff --git a/src/partners/wyre.ts b/src/partners/wyre.ts index 2459b9fd..8f43c335 100644 --- a/src/partners/wyre.ts +++ b/src/partners/wyre.ts @@ -121,6 +121,9 @@ export function processWyreTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: safeParseFloat(tx.sourceAmount), direction, exchangeType: 'fiat', @@ -128,6 +131,9 @@ export function processWyreTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: safeParseFloat(tx.destAmount), timestamp: dateMs / 1000, isoDate: date.toISOString(), diff --git a/src/partners/xanpool.ts b/src/partners/xanpool.ts index 4ca1288c..ce23ebcf 100644 --- a/src/partners/xanpool.ts +++ b/src/partners/xanpool.ts @@ -11,7 +11,7 @@ import { import fetch from 'node-fetch' import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' -import { datelog, smartIsoDateFromTimestamp } from '../util' +import { smartIsoDateFromTimestamp } from '../util' const asXanpoolTx = asObject({ id: asString, @@ -51,6 +51,7 @@ const LIMIT = 100 const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 5 // 5 days async function queryXanpool(pluginParams: PluginParams): Promise { + const { log } = pluginParams const { settings, apiKeys } = asXanpoolPluginParams(pluginParams) const { apiKey, apiSecret } = apiKeys let offset = 0 @@ -69,7 +70,7 @@ async function queryXanpool(pluginParams: PluginParams): Promise { let done = false while (!done) { let oldestIsoDate = '999999999999999999999999999999999999' - datelog(`Query Xanpool offset: ${offset}`) + log(`Query offset: ${offset}`) const response = await fetch( `https://${apiKey}:${apiSecret}@xanpool.com/api/v2/transactions?pageSize=${LIMIT}&page=${offset}` @@ -78,7 +79,7 @@ async function queryXanpool(pluginParams: PluginParams): Promise { const txs = asXanpoolResult(result).data if (txs.length === 0) { - datelog(`ChangeHero done at offset ${offset}`) + log(`Done at offset ${offset}`) break } for (const rawTx of txs) { @@ -91,17 +92,15 @@ async function queryXanpool(pluginParams: PluginParams): Promise { oldestIsoDate = standardTx.isoDate } if (standardTx.isoDate < previousLatestIsoDate && !done) { - datelog( - `Xanpool done: date ${standardTx.isoDate} < ${previousLatestIsoDate}` - ) + log(`Done: date ${standardTx.isoDate} < ${previousLatestIsoDate}`) done = true } } - datelog(`oldestIsoDate ${oldestIsoDate}`) + log(`oldestIsoDate ${oldestIsoDate}`) offset += LIMIT } } catch (e) { - datelog(e) + log.error(String(e)) } const out = { settings: { @@ -130,6 +129,9 @@ export function processXanpoolTx(rawTx: unknown): StandardTx { depositTxid: undefined, depositAddress: undefined, depositCurrency: tx.currency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.fiat, direction: 'buy', exchangeType: 'fiat', @@ -137,6 +139,9 @@ export function processXanpoolTx(rawTx: unknown): StandardTx { payoutTxid: tx.blockchainTxId, payoutAddress: tx.wallet, payoutCurrency: tx.cryptoCurrency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.crypto, timestamp: smartIsoDateFromTimestamp(new Date(tx.createdAt).getTime()) .timestamp, @@ -152,6 +157,9 @@ export function processXanpoolTx(rawTx: unknown): StandardTx { depositTxid: tx.blockchainTxId, depositAddress: Object.values(tx.depositWallets ?? {})[0], depositCurrency: tx.cryptoCurrency, + depositChainPluginId: undefined, + depositEvmChainId: undefined, + depositTokenId: undefined, depositAmount: tx.crypto, direction: 'sell', exchangeType: 'fiat', @@ -159,6 +167,9 @@ export function processXanpoolTx(rawTx: unknown): StandardTx { payoutTxid: undefined, payoutAddress: undefined, payoutCurrency: tx.currency, + payoutChainPluginId: undefined, + payoutEvmChainId: undefined, + payoutTokenId: undefined, payoutAmount: tx.fiat, timestamp: smartIsoDateFromTimestamp(new Date(tx.createdAt).getTime()) .timestamp, diff --git a/src/queryEngine.ts b/src/queryEngine.ts index 5c3dada6..2f1e1828 100644 --- a/src/queryEngine.ts +++ b/src/queryEngine.ts @@ -1,3 +1,4 @@ +import { Semaphore } from 'async-mutex' import nano from 'nano' import { config } from './config' @@ -23,6 +24,7 @@ import { lifi } from './partners/lifi' import { moonpay } from './partners/moonpay' import { paybis } from './partners/paybis' import { paytrie } from './partners/paytrie' +import { rango } from './partners/rango' import { safello } from './partners/safello' import { sideshift } from './partners/sideshift' import { simplex } from './partners/simplex' @@ -32,8 +34,23 @@ import { maya, thorchain } from './partners/thorchain' import { transak } from './partners/transak' import { wyre } from './partners/wyre' import { xanpool } from './partners/xanpool' -import { asApp, asApps, asProgressSettings, DbTx, StandardTx } from './types' -import { datelog, promiseTimeout, standardizeNames } from './util' +import { + asApp, + asApps, + asDisablePartnerQuery, + asProgressSettings, + DbTx, + DisablePartnerQuery, + ScopedLog, + StandardTx +} from './types' +import { createScopedLog, promiseTimeout, standardizeNames } from './util' + +/** Local datelog for engine-level logs not associated with a specific app/partner */ +const datelog = (...args: unknown[]): void => { + const date = new Date().toISOString() + console.log(date, ...args) +} const nanoDb = nano(config.couchDbFullpath) @@ -60,6 +77,7 @@ const plugins = [ moonpay, paybis, paytrie, + rango, safello, sideshift, simplex, @@ -76,12 +94,27 @@ const BULK_FETCH_SIZE = 50 const snooze: Function = async (ms: number) => await new Promise((resolve: Function) => setTimeout(resolve, ms)) -export async function queryEngine(): Promise { - const dbProgress = nanoDb.db.use('reports_progresscache') - const dbApps = nanoDb.db.use('reports_apps') +const dbProgress = nanoDb.db.use('reports_progresscache') +const dbApps = nanoDb.db.use('reports_apps') +const dbSettings: nano.DocumentScope = nanoDb.db.use( + 'reports_settings' +) +export async function queryEngine(): Promise { while (true) { datelog('Starting query loop...') + let disablePartnerQuery: DisablePartnerQuery = { + plugins: {}, + appPartners: {} + } + try { + const disablePartnerQueryDoc = await dbSettings.get('disablePartnerQuery') + if (disablePartnerQueryDoc != null) { + disablePartnerQuery = asDisablePartnerQuery(disablePartnerQueryDoc) + } + } catch (e) { + datelog('Error getting disablePartnerQuery', e) + } // get the contents of all reports_apps docs const query = { selector: { @@ -91,62 +124,96 @@ export async function queryEngine(): Promise { } const rawApps = await dbApps.find(query) const apps = asApps(rawApps.docs) - let promiseArray: Array> = [] - let remainingPartners: String[] = [] // loop over every app for (const app of apps) { + const semaphore = new Semaphore(MAX_CONCURRENT_QUERIES) if (config.soloAppIds != null && !config.soloAppIds.includes(app.appId)) { continue } let partnerStatus: string[] = [] + const runPlugins: RunPluginParams[] = [] + let remainingPlugins: RunPluginParams[] = [] // loop over every pluginId that app uses - remainingPartners = Object.keys(app.partnerIds) for (const partnerId in app.partnerIds) { const pluginId = app.partnerIds[partnerId].pluginId ?? partnerId - - if ( - config.soloPartnerIds != null && - !config.soloPartnerIds.includes(partnerId) - ) { - continue + if (config.soloPartnerIds?.includes(partnerId) !== true) { + if (disablePartnerQuery.plugins[pluginId]) { + continue + } + const appPartnerId = `${app.appId}_${partnerId}` + if (disablePartnerQuery.appPartners[appPartnerId]) { + continue + } + if ( + config.soloPartnerIds != null && + !config.soloPartnerIds.includes(partnerId) + ) { + continue + } } - remainingPartners.push(partnerId) - promiseArray.push( - runPlugin(app, partnerId, pluginId, dbProgress).finally(() => { - remainingPartners = remainingPartners.filter( - string => string !== partnerId + const runPluginParams: RunPluginParams = { app, partnerId, pluginId } + runPlugins.push(runPluginParams) + remainingPlugins.push(runPluginParams) + } + const promises: Array> = [] + for (const runPluginParams of runPlugins) { + await semaphore.acquire() + const promise = runPlugin(runPluginParams) + .then(status => { + partnerStatus = [...partnerStatus, status] + }) + .finally(() => { + semaphore.release() + // remove the plugin from the remaining plugins + remainingPlugins = remainingPlugins.filter( + plugin => plugin !== runPluginParams ) - if (remainingPartners.length > 0) { + if (remainingPlugins.length > 0) { datelog( `REMAINING PLUGINS for ${app.appId}:`, - remainingPartners.join(', ') + remainingPlugins.map(plugin => plugin.partnerId).join(', ') ) } }) - ) - if (promiseArray.length >= MAX_CONCURRENT_QUERIES) { - const status = await Promise.all(promiseArray) - // log how long every app + plugin took to run - datelog(status) - partnerStatus = [...partnerStatus, ...status] - promiseArray = [] - } + promises.push(promise) } - datelog(partnerStatus) + await Promise.all(promises) + datelog(partnerStatus.join('\n')) } - const partnerStatus = await Promise.all(promiseArray) - // log how long every app + plugin took to run - datelog(partnerStatus) datelog(`Snoozing for ${QUERY_FREQ_MS} milliseconds`) await snooze(QUERY_FREQ_MS) } } +const checkUpdateTx = ( + oldTx: StandardTx, + newTx: StandardTx +): string[] | undefined => { + const changedFields: string[] = [] + + if (oldTx.status !== newTx.status) changedFields.push('status') + if (oldTx.depositChainPluginId !== newTx.depositChainPluginId) + changedFields.push('depositChainPluginId') + if (oldTx.depositEvmChainId !== newTx.depositEvmChainId) + changedFields.push('depositEvmChainId') + if (oldTx.depositTokenId !== newTx.depositTokenId) + changedFields.push('depositTokenId') + if (oldTx.payoutChainPluginId !== newTx.payoutChainPluginId) + changedFields.push('payoutChainPluginId') + if (oldTx.payoutEvmChainId !== newTx.payoutEvmChainId) + changedFields.push('payoutEvmChainId') + if (oldTx.payoutTokenId !== newTx.payoutTokenId) + changedFields.push('payoutTokenId') + + return changedFields.length > 0 ? changedFields : undefined +} + const filterAddNewTxs = async ( pluginId: string, dbTransactions: nano.DocumentScope, docIds: string[], - transactions: StandardTx[] + transactions: StandardTx[], + log: ScopedLog ): Promise => { if (docIds.length < 1 || transactions.length < 1) return const queryResults = await dbTransactions.fetch( @@ -165,7 +232,11 @@ const filterAddNewTxs = async ( throw new Error(`Cant find tx from docId ${docId}`) } - if (queryResult == null) { + if ( + queryResult == null || + !('doc' in queryResult) || + queryResult.doc == null + ) { // Get the full transaction const newObj = { _id: docId, _rev: undefined, ...tx } @@ -173,32 +244,40 @@ const filterAddNewTxs = async ( newObj.depositCurrency = standardizeNames(newObj.depositCurrency) newObj.payoutCurrency = standardizeNames(newObj.payoutCurrency) - datelog(`new doc id: ${newObj._id}`) + log(`[filterAddNewTxs] new doc id: ${newObj._id}`) newDocs.push(newObj) } else { - if ('doc' in queryResult) { - if (tx.status !== queryResult.doc?.status) { - const oldStatus = queryResult.doc?.status - const newStatus = tx.status - const newObj = { _id: docId, _rev: queryResult.doc?._rev, ...tx } - newDocs.push(newObj) - datelog(`updated doc id: ${newObj._id} ${oldStatus} -> ${newStatus}`) - } + const changedFields = checkUpdateTx(queryResult.doc, tx) + if (changedFields != null) { + const oldStatus = queryResult.doc?.status + const newStatus = tx.status + const newObj = { _id: docId, _rev: queryResult.doc?._rev, ...tx } + newDocs.push(newObj) + log( + `[filterAddNewTxs] updated doc id: ${ + newObj._id + } ${oldStatus} -> ${newStatus} [${changedFields.join(', ')}]` + ) } } } try { - await promiseTimeout('pagination', pagination(newDocs, dbTransactions)) + await promiseTimeout( + 'pagination', + pagination(newDocs, dbTransactions, log), + log + ) } catch (e) { - datelog('Error doing bulk transaction insert', e) + log.error('[filterAddNewTxs] Error doing bulk transaction insert', e) throw e } } async function insertTransactions( transactions: StandardTx[], - pluginId: string + pluginId: string, + log: ScopedLog ): Promise { const dbTransactions: nano.DocumentScope = nanoDb.db.use( 'reports_transactions' @@ -214,47 +293,44 @@ async function insertTransactions( // Collect a batch of docIds if (docIds.length < BULK_FETCH_SIZE) continue - datelog( - `insertTransactions ${startIndex} to ${i} of ${transactions.length}` - ) - await filterAddNewTxs(pluginId, dbTransactions, docIds, transactions) + log(`[insertTransactions] ${startIndex} to ${i} of ${transactions.length}`) + await filterAddNewTxs(pluginId, dbTransactions, docIds, transactions, log) docIds = [] startIndex = i + 1 } - await filterAddNewTxs(pluginId, dbTransactions, docIds, transactions) + await filterAddNewTxs(pluginId, dbTransactions, docIds, transactions, log) } -async function runPlugin( - app: ReturnType, - partnerId: string, - pluginId: string, - dbProgress: nano.DocumentScope -): Promise { +interface RunPluginParams { + app: ReturnType + partnerId: string + pluginId: string +} + +async function runPlugin(params: RunPluginParams): Promise { + const { app, partnerId, pluginId } = params const start = Date.now() + const log = createScopedLog(app.appId, partnerId) let errorText = '' try { // obtains function that corresponds to current pluginId const plugin = plugins.find(plugin => plugin.pluginId === pluginId) // if current plugin is not within the list of partners skip to next if (plugin === undefined) { - errorText = `Missing or disabled plugin ${app.appId.toLowerCase()}_${partnerId}` - datelog(errorText) + errorText = `[runPlugin] ${partnerId} Missing or disabled plugin` + log(errorText) return errorText } // get progress cache to see where previous query ended - datelog( - `Starting with partner:${partnerId} plugin:${pluginId}, app: ${app.appId}` - ) + log(`[runPlugin] Starting with plugin:${pluginId}`) const progressCacheFileName = `${app.appId.toLowerCase()}:${partnerId}` const out = await dbProgress.get(progressCacheFileName).catch(e => { if (e.error != null && e.error === 'not_found') { - datelog( - `Previous Progress Record Not Found ${app.appId.toLowerCase()}_${partnerId}` - ) + log(`[runPlugin] Previous Progress Record Not Found`) return {} } else { - console.log(e) + log.error('[runPlugin] Error fetching progress', e) } }) @@ -273,35 +349,43 @@ async function runPlugin( // set apiKeys and settings for use in partner's function const { apiKeys } = app.partnerIds[partnerId] const settings = progressSettings.progressCache - datelog(`Querying ${app.appId.toLowerCase()}_${partnerId}`) + log(`[runPlugin] Querying`) // run the plugin function const result = await promiseTimeout( 'queryFunc', plugin.queryFunc({ apiKeys, - settings - }) + settings, + log + }), + log ) - datelog(`Successful query: ${app.appId.toLowerCase()}_${partnerId}`) + log(`[runPlugin] Successful query`) await promiseTimeout( 'insertTransactions', - insertTransactions(result.transactions, `${app.appId}_${partnerId}`) - ) + insertTransactions(result.transactions, `${app.appId}_${partnerId}`, log), + log + ).catch(e => { + throw new Error(`Error inserting transactions: ${String(e)}`) + }) progressSettings.progressCache = result.settings progressSettings._id = progressCacheFileName await promiseTimeout( 'dbProgress.insert', - dbProgress.insert(progressSettings) - ) + dbProgress.insert(progressSettings), + log + ).catch(e => { + throw new Error(`Error inserting progress: ${String(e)}`) + }) // Returning a successful completion message const completionTime = (Date.now() - start) / 1000 - const successfulCompletionMessage = `Successful update: ${app.appId.toLowerCase()}_${partnerId} in ${completionTime} seconds.` - datelog(successfulCompletionMessage) + const successfulCompletionMessage = `[runPlugin] ${partnerId} Successful update in ${completionTime} seconds.` + log(successfulCompletionMessage) return successfulCompletionMessage } catch (e) { - errorText = `Error: ${app.appId.toLowerCase()}_${partnerId}. Error message: ${e}` - datelog(errorText) + errorText = `[runPlugin] ${partnerId} Error: ${String(e)}` + log.error(errorText) return errorText } } diff --git a/src/ratesEngine.ts b/src/ratesEngine.ts index cbc30463..f5b56c1c 100644 --- a/src/ratesEngine.ts +++ b/src/ratesEngine.ts @@ -6,14 +6,24 @@ import { config } from './config' import { asDbCurrencyCodeMappings, asDbTx, + asV3RatesParams, CurrencyCodeMappings, - DbTx + DbTx, + V3RatesParams } from './types' import { datelog, safeParseFloat, standardizeNames } from './util' +import { isFiatCurrency } from './util/fiatCurrency' const nanoDb = nano(config.couchDbFullpath) -const QUERY_FREQ_MS = 3000 -const QUERY_LIMIT = 10 +const QUERY_FREQ_MS = 2000 +const QUERY_LIMIT = 20 +const RATES_SERVERS = [ + 'https://rates1.edge.app', + 'https://rates2.edge.app', + 'https://rates3.edge.app', + 'https://rates4.edge.app' +] + const snooze: Function = async (ms: number) => await new Promise((resolve: Function) => setTimeout(resolve, ms)) @@ -45,7 +55,7 @@ export async function ratesEngine(): Promise { limit: QUERY_LIMIT } ] - let bookmark + const bookmarks: Array = [] let count = 1 while (true) { count++ @@ -53,17 +63,19 @@ export async function ratesEngine(): Promise { const result2 = await dbSettings.get('currencyCodeMappings') const { mappings } = asDbCurrencyCodeMappings(result2) - const query = queries[count % 2] - query.bookmark = bookmark + const index = count % 2 + + const query = queries[index] + query.bookmark = bookmarks[index] const result = await dbTransactions.find(query) if ( typeof result.bookmark === 'string' && result.docs.length === QUERY_LIMIT ) { - bookmark = result.bookmark + bookmarks[index] = result.bookmark } else { - bookmark = undefined + bookmarks[index] = undefined } try { asDbQueryResult(result) @@ -98,19 +110,195 @@ export async function ratesEngine(): Promise { } catch (e) { datelog('Error doing bulk usdValue insert', e) } - if (bookmark == null) { + if (bookmarks[index] == null) { datelog(`Snoozing for ${QUERY_FREQ_MS} milliseconds`) await snooze(QUERY_FREQ_MS) } else { - datelog(`Fetching bookmark ${bookmark}`) + datelog(`Fetching bookmark ${bookmarks[index]}`) + } + } +} + +async function updateTxValuesV3(transaction: DbTx): Promise { + const { + isoDate, + depositCurrency, + depositChainPluginId, + depositTokenId, + depositAmount, + payoutChainPluginId, + payoutTokenId, + payoutCurrency, + payoutAmount + } = transaction + + let depositIsFiat = false + let payoutIsFiat = false + const ratesRequest: V3RatesParams = { + targetFiat: 'USD', + crypto: [], + fiat: [] + } + if (depositChainPluginId != null && depositTokenId !== undefined) { + ratesRequest.crypto.push({ + isoDate: new Date(isoDate), + asset: { + pluginId: depositChainPluginId, + tokenId: depositTokenId + }, + rate: undefined + }) + } else if (isFiatCurrency(depositCurrency) && depositCurrency !== 'USD') { + depositIsFiat = true + ratesRequest.fiat.push({ + isoDate: new Date(isoDate), + fiatCode: depositCurrency, + rate: undefined + }) + } else if (depositCurrency !== 'USD') { + console.error( + `Deposit asset is not a crypto asset or fiat currency ${depositCurrency} ${depositChainPluginId} ${depositTokenId}` + ) + return + } + + if (payoutChainPluginId != null && payoutTokenId !== undefined) { + ratesRequest.crypto.push({ + isoDate: new Date(isoDate), + asset: { + pluginId: payoutChainPluginId, + tokenId: payoutTokenId + }, + rate: undefined + }) + } else if (isFiatCurrency(payoutCurrency) && payoutCurrency !== 'USD') { + payoutIsFiat = true + ratesRequest.fiat.push({ + isoDate: new Date(isoDate), + fiatCode: payoutCurrency, + rate: undefined + }) + } else if (payoutCurrency !== 'USD') { + console.error( + `Payout asset is not a crypto asset or fiat currency ${payoutCurrency} ${payoutChainPluginId} ${payoutTokenId}` + ) + return + } + + const server = RATES_SERVERS[Math.floor(Math.random() * RATES_SERVERS.length)] + datelog(`Getting v3 rates from ${server}`) + const ratesResponse = await fetch(`${server}/v3/rates`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(ratesRequest) + }) + const ratesResponseJson = await ratesResponse.json() + const rates = asV3RatesParams(ratesResponseJson) + const depositRateObf = depositIsFiat + ? rates.fiat.find(rate => rate.fiatCode === depositCurrency) + : rates.crypto.find( + rate => + rate.asset.pluginId === depositChainPluginId && + (rate.asset.tokenId ?? null) === depositTokenId + ) + const payoutRateObf = payoutIsFiat + ? rates.fiat.find(rate => rate.fiatCode === payoutCurrency) + : rates.crypto.find( + rate => + rate.asset.pluginId === payoutChainPluginId && + (rate.asset.tokenId ?? null) === payoutTokenId + ) + + const depositRate = depositRateObf?.rate + const payoutRate = payoutRateObf?.rate + + let changed = false + // Calculate and fill out payoutAmount if it is zero + if (payoutAmount === 0) { + if (depositRate == null) { + console.error( + `No rate found for deposit ${depositCurrency} ${depositChainPluginId} ${depositTokenId}` + ) + } + + if (payoutRate == null) { + console.error( + `No rate found for payout ${payoutCurrency} ${payoutChainPluginId} ${payoutTokenId}` + ) + } + if (depositRate != null && payoutRate != null) { + transaction.payoutAmount = (depositAmount * depositRate) / payoutRate + changed = true } } + + // Calculate the usdValue first trying to use the deposit amount. If that's not available + // then try to use the payout amount. + const t = transaction + if (transaction.usdValue == null || transaction.usdValue <= 0) { + if (depositRate != null) { + transaction.usdValue = depositAmount * depositRate + changed = true + datelog( + `V3 SUCCESS id:${t._id} ${t.isoDate.slice(0, 10)} deposit:${ + t.depositCurrency + }-${t.depositChainPluginId}-${ + t.depositTokenId + } rate:${depositRate} usdValue:${t.usdValue}` + ) + } else if (payoutRate != null) { + transaction.usdValue = transaction.payoutAmount * payoutRate + changed = true + datelog( + `V3 SUCCESS id:${t._id} ${t.isoDate.slice(0, 10)} payout:${ + t.payoutCurrency + }-${t.payoutChainPluginId}-${ + t.payoutTokenId + } rate:${payoutRate} usdValue:${t.usdValue}` + ) + } + } + if (!changed) { + datelog( + `V3 NO CHANGE id:${t._id} ${t.isoDate.slice(0, 10)} ${ + t.depositCurrency + } ${t.payoutCurrency}` + ) + transaction._id = undefined + } } -export async function updateTxValues( +async function updateTxValues( transaction: DbTx, mappings: CurrencyCodeMappings ): Promise { + if ( + transaction.depositChainPluginId != null && + transaction.depositTokenId !== undefined && + transaction.payoutChainPluginId != null && + transaction.payoutTokenId !== undefined + ) { + return await updateTxValuesV3(transaction) + } + + if ( + transaction.depositChainPluginId != null && + transaction.depositTokenId !== undefined && + isFiatCurrency(transaction.payoutCurrency) + ) { + return await updateTxValuesV3(transaction) + } + + if ( + isFiatCurrency(transaction.depositCurrency) && + transaction.payoutChainPluginId != null && + transaction.payoutTokenId !== undefined + ) { + return await updateTxValuesV3(transaction) + } + let success = false const date: string = transaction.isoDate if (mappings[transaction.depositCurrency] != null) { @@ -217,9 +405,19 @@ async function getExchangeRate( retry: number = 0 ): Promise { const hourDate = dateRoundDownHour(date) - const currencyA = standardizeNames(ca) - const currencyB = standardizeNames(cb) - const url = `https://rates2.edge.app/v1/exchangeRate?currency_pair=${currencyA}_${currencyB}&date=${hourDate}` + let currencyA = standardizeNames(ca) + let currencyB = standardizeNames(cb) + + if (currencyA === currencyB) { + return 1 + } + + currencyA = isFiatCurrency(currencyA) ? `iso:${currencyA}` : currencyA + currencyB = isFiatCurrency(currencyB) ? `iso:${currencyB}` : currencyB + + const server = RATES_SERVERS[Math.floor(Math.random() * RATES_SERVERS.length)] + const url = `${server}/v2/exchangeRate?currency_pair=${currencyA}_${currencyB}&date=${hourDate}` + datelog(`Getting v2 exchange rate from ${server}`) try { const result = await fetch(url, { method: 'GET' }) if (!result.ok) { diff --git a/src/types.ts b/src/types.ts index 24176c90..8e1bb5c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,7 +1,10 @@ import { asArray, + asBoolean, + asDate, asEither, asMap, + asMaybe, asNull, asNumber, asObject, @@ -19,6 +22,13 @@ export const asPluginParams = asObject({ settings: asMap((raw: any): any => raw), apiKeys: asMap((raw: any): any => raw) }) + +/** Scoped logging interface passed to plugins */ +export interface ScopedLog { + (message: string, ...args: unknown[]): void + warn: (message: string, ...args: unknown[]) => void + error: (message: string, ...args: unknown[]) => void +} export interface PluginResult { // copy the type from standardtx from reports transactions: StandardTx[] @@ -34,11 +44,15 @@ export interface PartnerPlugin { const asStatus = asValue( 'complete', + 'confirming', + 'withdrawing', 'processing', 'pending', 'expired', 'blocked', 'refunded', + 'cancelled', + 'failed', 'other' ) @@ -85,6 +99,7 @@ const asFiatPaymentType = asValue( 'moonpaybalance', 'neft', 'neteller', + 'ozow', 'payid', 'paynow', 'paypal', @@ -115,6 +130,9 @@ export const asStandardTx = asObject({ depositTxid: asOptional(asString), depositAddress: asOptional(asString), depositCurrency: asString, + depositChainPluginId: asOptional(asString), + depositEvmChainId: asOptional(asNumber), + depositTokenId: asOptional(asEither(asString, asNull)), depositAmount: asSafeNumber, direction: asOptional(asDirection), exchangeType: asOptional(asExchangeType), @@ -122,6 +140,9 @@ export const asStandardTx = asObject({ payoutTxid: asOptional(asString), payoutAddress: asOptional(asString), payoutCurrency: asString, + payoutChainPluginId: asOptional(asString), + payoutEvmChainId: asOptional(asNumber), + payoutTokenId: asOptional(asEither(asString, asNull)), payoutAmount: asSafeNumber, status: asStatus, isoDate: asString, @@ -204,6 +225,37 @@ export const asAnalyticsResult = asObject({ end: asNumber }) +// v3/rates response cleaner (matches GUI's shape) +const asV3CryptoAsset = asObject({ + pluginId: asString, + tokenId: asOptional(asEither(asString, asNull)) +}) +const asV3CryptoRate = asObject({ + isoDate: asOptional(asDate), + asset: asV3CryptoAsset, + rate: asOptional(asNumber) +}) +const asV3FiatRate = asObject({ + isoDate: asOptional(asDate), + fiatCode: asString, + rate: asOptional(asNumber) +}) +export const asV3RatesParams = asObject({ + targetFiat: asString, + crypto: asArray(asV3CryptoRate), + fiat: asArray(asV3FiatRate) +}) + +export const asDisablePartnerQuery = asMaybe( + asObject({ + plugins: asObject(asBoolean), + appPartners: asObject(asBoolean) + }), + { plugins: {}, appPartners: {} } +) + +export type DisablePartnerQuery = ReturnType +export type V3RatesParams = ReturnType export type Bucket = ReturnType export type AnalyticsResult = ReturnType @@ -211,5 +263,7 @@ export type CurrencyCodeMappings = ReturnType export type DbCurrencyCodeMappings = ReturnType export type DbTx = ReturnType export type StandardTx = ReturnType -export type PluginParams = ReturnType +export type PluginParams = ReturnType & { + log: ScopedLog +} export type Status = ReturnType diff --git a/src/util.ts b/src/util.ts index e17a1ce5..e6beffb5 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,6 +1,7 @@ import fetch, { RequestInfo, RequestInit, Response } from 'node-fetch' import { config } from './config' +import { ScopedLog } from './types' export const SIX_DAYS = 6 @@ -48,11 +49,12 @@ export const standardizeNames = (field: string): string => { export const promiseTimeout = async ( msg: string, - p: Promise + p: Promise, + log: ScopedLog ): Promise => { const timeoutMins = config.timeoutOverrideMins ?? 5 return await new Promise((resolve, reject) => { - datelog('STARTING', msg) + log(`STARTING ${msg}`) setTimeout(() => reject(new Error(`Timeout: ${msg}`)), 60000 * timeoutMins) p.then(v => resolve(v)).catch(e => reject(e)) }) @@ -79,11 +81,39 @@ export const smartIsoDateFromTimestamp = ( } } +/** Datelog for non-partner files. Partners should use the scoped log passed via PluginParams. */ export const datelog = function(...args: any): void { const date = new Date().toISOString() console.log(date, ...args) } +/** + * Creates a scoped logger that prefixes all messages with ISO date and app_partnerId + */ +export const createScopedLog = ( + appId: string, + partnerId: string +): ScopedLog => { + const prefix = `${appId.toLowerCase()}_${partnerId}` + + const log = (message: string, ...args: unknown[]): void => { + const date = new Date().toISOString() + console.log(date, prefix, message, ...args) + } + + log.warn = (message: string, ...args: unknown[]): void => { + const date = new Date().toISOString() + console.warn(date, prefix, message, ...args) + } + + log.error = (message: string, ...args: unknown[]): void => { + const date = new Date().toISOString() + console.error(date, prefix, message, ...args) + } + + return log +} + export const snoozeReject = async (ms: number): Promise => await new Promise((resolve: Function, reject: Function) => setTimeout(reject, ms) diff --git a/src/util/asEdgeTokenId.ts b/src/util/asEdgeTokenId.ts index 9b87efaf..c3125979 100644 --- a/src/util/asEdgeTokenId.ts +++ b/src/util/asEdgeTokenId.ts @@ -2,3 +2,156 @@ import { asEither, asNull, asString } from 'cleaners' export type EdgeTokenId = string | null export const asEdgeTokenId = asEither(asString, asNull) + +export type TokenType = + | 'simple' + | 'evm' + | 'cosmos' + | 'xrpl' + | 'colon-delimited' + | 'lowercase' + | null + +export const tokenTypes: Record = { + abstract: 'evm', + algorand: 'simple', + arbitrum: 'evm', + avalanche: 'evm', + axelar: 'cosmos', + base: 'evm', + binance: null, + binancesmartchain: 'evm', + bitcoin: null, + bitcoincash: null, + bitcoingold: null, + bitcoinsv: null, + bobevm: 'evm', + botanix: 'evm', + cardano: null, + celo: 'evm', + coreum: 'cosmos', + cosmoshub: 'cosmos', + dash: null, + digibyte: null, + dogecoin: null, + eboost: null, + ecash: null, + eos: null, + ethereum: 'evm', + ethereumclassic: 'evm', + ethereumpow: 'evm', + fantom: 'evm', + feathercoin: null, + filecoin: null, + filecoinfevm: 'evm', + fio: null, + groestlcoin: null, + hedera: null, + hyperevm: 'evm', + liberland: 'simple', + litecoin: null, + monero: null, + optimism: 'evm', + osmosis: 'cosmos', + piratechain: null, + pivx: null, + polkadot: null, + polygon: 'evm', + pulsechain: 'evm', + qtum: null, + ravencoin: null, + ripple: 'xrpl', + rsk: 'evm', + smartcash: null, + solana: 'simple', + sonic: 'evm', + stellar: null, + sui: 'colon-delimited', + telos: null, + tezos: null, + thorchainrune: 'cosmos', + ton: null, + tron: 'simple', + ufo: null, + vertcoin: null, + wax: null, + zano: 'lowercase', + zcash: null, + zcoin: null, + zksync: 'evm' +} + +export type CurrencyCodeToAssetMapping = Record< + string, + { pluginId: string; tokenId: EdgeTokenId } +> + +// TODO: Remove this once we've migrated to using moonpayNetworkToPluginId functions instead +export type ChainNameToPluginIdMapping = Record + +export const createTokenId = ( + pluginType: TokenType, + currencyCode: string, + contractAddress?: string | null +): EdgeTokenId => { + if (contractAddress == null) { + return null + } + switch (pluginType) { + // Use contract address as-is: + case 'simple': { + return contractAddress + } + + // EVM token support: + case 'evm': { + return contractAddress.toLowerCase().replace(/^0x/, '') + } + + // Cosmos token support: + case 'cosmos': { + // Regexes inspired by a general regex in https://github.com/cosmos/cosmos-sdk + // Broken up to more tightly enforce the rules for each type of asset so the entered value matches what a node would expect + const ibcDenomRegex = /^ibc\/[0-9A-F]{64}$/ + const nativeDenomRegex = /^(?!ibc)[a-z][a-z0-9/]{2,127}/ + + if ( + contractAddress == null || + (!ibcDenomRegex.test(contractAddress) && + !nativeDenomRegex.test(contractAddress)) + ) { + throw new Error('Invalid contract address') + } + + return contractAddress.toLowerCase().replace(/\//g, '') + } + + // XRP token support: + case 'xrpl': { + let currency: string + if (currencyCode.length > 3) { + const hexCode = Buffer.from(currencyCode, 'utf8').toString('hex') + currency = hexCode.toUpperCase().padEnd(40, '0') + } else { + currency = currencyCode + } + + return `${currency}-${contractAddress}` + } + + // Sui token support: + case 'colon-delimited': { + return contractAddress.replace(/:/g, '') + } + + case 'lowercase': { + return contractAddress.toLowerCase() + } + + default: { + // No token support: + // these chains don't support tokens + throw new Error('Tokens are not supported for this chain') + } + } +} diff --git a/src/util/chainIds.ts b/src/util/chainIds.ts new file mode 100644 index 00000000..2f08d1f7 --- /dev/null +++ b/src/util/chainIds.ts @@ -0,0 +1,32 @@ +export const EVM_CHAIN_IDS: Record = { + abstract: 2741, + arbitrum: 42161, + avalanche: 43114, + base: 8453, + binancesmartchain: 56, + bobevm: 60808, + botanix: 3637, + celo: 42220, + ethereum: 1, + ethereumclassic: 61, + ethereumpow: 10001, + fantom: 250, + filecoinfevm: 314, + hyperevm: 999, + optimism: 10, + polygon: 137, + pulsechain: 369, + rsk: 30, + sonic: 146, + zksync: 324 +} + +export const REVERSE_EVM_CHAIN_IDS: Record = Object.entries( + EVM_CHAIN_IDS +).reduce((acc, [key, value]) => { + acc[value] = key + return acc +}, {}) + +export const reverseEvmChainId = (evmChainId?: number): string | undefined => + evmChainId != null ? REVERSE_EVM_CHAIN_IDS[evmChainId] : undefined diff --git a/yarn.lock b/yarn.lock index d7baa4bc..deec5370 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1533,6 +1533,13 @@ async-limiter@~1.0.0: resolved "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz" integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== +async-mutex@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.5.0.tgz#353c69a0b9e75250971a64ac203b0ebfddd75482" + integrity sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA== + dependencies: + tslib "^2.4.0" + asynciterator.prototype@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/asynciterator.prototype/-/asynciterator.prototype-1.0.0.tgz#8c5df0514936cdd133604dfcc9d3fb93f09b2b62"