Prevent future-dated request from filling dbs

src/v3/getRates.ts

@@ -86,11 +86,12 @@ const queryProviders = async (
 
 const updateProviders = async (
   providers: RateProvider[],
-  rates: UpdateRatesParams
+  rates: UpdateRatesParams,
+  rightNow: Date
 ): Promise<void> => {
   for (const p of providers) {
     if (p.updateRates != null) {
-      p.updateRates(rates).catch(err => {
+      p.updateRates(rates, rightNow).catch(err => {
         console.error(`Error updating rates from ${p.providerId}`, err)
       })
     }
@@ -152,20 +153,28 @@ export const getRates: GetRatesFunc = async (params, rightNow) => {
   )
 
   // Update redis with db and api data
-  updateProviders(memoryProviders, {
-    targetFiat,
-    crypto: new Map([...dbResults.foundCrypto, ...apiResults.foundCrypto]),
-    fiat: new Map([...dbResults.foundFiat, ...apiResults.foundFiat])
-  }).catch(e => {
+  updateProviders(
+    memoryProviders,
+    {
+      targetFiat,
+      crypto: new Map([...dbResults.foundCrypto, ...apiResults.foundCrypto]),
+      fiat: new Map([...dbResults.foundFiat, ...apiResults.foundFiat])
+    },
+    rightNow
+  ).catch(e => {
     console.error('Error updating memoryproviders', e)
   })
 
   // Update db with api data
-  updateProviders(dbProviders, {
-    targetFiat,
-    crypto: apiResults.foundCrypto,
-    fiat: apiResults.foundFiat
-  }).catch(e => {
+  updateProviders(
+    dbProviders,
+    {
+      targetFiat,
+      crypto: apiResults.foundCrypto,
+      fiat: apiResults.foundFiat
+    },
+    rightNow
+  ).catch(e => {
     console.error('Error updating dbproviders', e)
   })
 

src/v3/providers/coinmarketcap/coinmarketcap.ts

@@ -10,7 +10,7 @@ import { asCouchDoc, syncedDocument } from 'edge-server-tools'
 
 import { config } from '../../../config'
 import { daysBetween, snooze } from '../../../utils/utils'
-import { TOKEN_TYPES_KEY } from '../../constants'
+import { FIVE_MINUTES, TOKEN_TYPES_KEY } from '../../constants'
 import {
   asStringNullMap,
   asTokenMap,
@@ -249,11 +249,11 @@ const getCurrentRates = async (ids: Set<string>): Promise<NumberMap> => {
 }
 const getHistoricalRates = async (
   ids: Set<string>,
-  date: string
+  date: string,
+  rightNow: Date
 ): Promise<NumberMap> => {
   const out: NumberMap = {}
-  const now = new Date()
-  const days = daysBetween(new Date(date), now)
+  const days = daysBetween(new Date(date), rightNow)
 
   // If we're querying a date more than 3 months in the past, use
   // daily average
@@ -277,6 +277,7 @@ const getHistoricalRates = async (
 
     const data = asCoinMarketCapHistoricalQuotes(json)
     for (const [key, value] of Object.entries(data.data)) {
+      if (value.quotes.length === 0) continue
       out[key] = value.quotes[0].quote.USD.price
     }
   } catch (e) {
@@ -320,15 +321,15 @@ export const coinmarketcap: RateProvider = {
     const allResults: RateBuckets = new Map()
     const promises: Array<Promise<void>> = []
     rateBuckets.forEach((ids, date) => {
-      if (isCurrent(new Date(date), rightNow)) {
+      if (isCurrent(new Date(date), rightNow, FIVE_MINUTES)) {
         promises.push(
           getCurrentRates(ids).then(results => {
             allResults.set(date, results)
           })
         )
       } else {
         promises.push(
-          getHistoricalRates(ids, date).then(results => {
+          getHistoricalRates(ids, date, rightNow).then(results => {
             allResults.set(date, results)
           })
         )

src/v3/providers/couch.ts

@@ -83,15 +83,15 @@ export const couch: RateProvider = {
       requestedRates: out.requestedRates
     }
   },
-  getFiatRates: async ({ targetFiat, requestedRates }) => {
+  getFiatRates: async ({ targetFiat, requestedRates }, rightNow) => {
     if (targetFiat !== 'USD') {
       return {
         foundRates: new Map(),
         requestedRates
       }
     }
 
-    const rateBuckets = reduceRequestedFiatRates(requestedRates)
+    const rateBuckets = reduceRequestedFiatRates(requestedRates, rightNow)
 
     const allResults: RateBuckets = new Map()
 
@@ -130,16 +130,19 @@ export const couch: RateProvider = {
       requestedRates: out.requestedRates
     }
   },
-  updateRates: async (params: UpdateRatesParams): Promise<void> => {
+  updateRates: async (
+    params: UpdateRatesParams,
+    rightNow: Date
+  ): Promise<void> => {
     if (params.targetFiat !== 'USD') {
       return
     }
     if (params.crypto.size === 0 && params.fiat.size === 0) {
       return
     }
 
-    const cryptoRateBuckets = groupCryptoRatesByTime(params.crypto)
-    const fiatRateBuckets = groupFiatRatesByTime(params.fiat)
+    const cryptoRateBuckets = groupCryptoRatesByTime(params.crypto, rightNow)
+    const fiatRateBuckets = groupFiatRatesByTime(params.fiat, rightNow)
 
     const ids = new Set<string>([
       ...cryptoRateBuckets.keys(),

src/v3/providers/currencyconverter.ts

@@ -65,21 +65,20 @@ const fetchCurrencyConverter = async (
 export const currencyconverter: RateProvider = {
   providerId: 'currencyconverter',
   type: 'api',
-  getFiatRates: async ({ targetFiat, requestedRates }) => {
+  getFiatRates: async ({ targetFiat, requestedRates }, rightNow) => {
     if (apiKey == null) {
       return {
         foundRates: new Map(),
         requestedRates
       }
     }
 
-    const rateBuckets = reduceRequestedFiatRates(requestedRates)
+    const rateBuckets = reduceRequestedFiatRates(requestedRates, rightNow)
 
-    const currentDate = new Date()
     const allResults: RateBuckets = new Map()
     const promises: Array<Promise<void>> = []
     rateBuckets.forEach((ids, date) => {
-      if (isCurrentFiat(new Date(date), currentDate)) {
+      if (isCurrentFiat(new Date(date), rightNow)) {
         promises.push(
           fetchCurrencyConverter(targetFiat, Array.from(ids)).then(results => {
             allResults.set(date, results)

src/v3/providers/redis.ts

@@ -81,15 +81,15 @@ export const redis: RateProvider = {
       requestedRates: out.requestedRates
     }
   },
-  getFiatRates: async ({ targetFiat, requestedRates }) => {
+  getFiatRates: async ({ targetFiat, requestedRates }, rightNow) => {
     if (targetFiat !== 'USD') {
       return {
         foundRates: new Map(),
         requestedRates
       }
     }
 
-    const rateBuckets = reduceRequestedFiatRates(requestedRates)
+    const rateBuckets = reduceRequestedFiatRates(requestedRates, rightNow)
 
     const allResults: RateBuckets = new Map()
     for (const [date, fiatPairs] of rateBuckets.entries()) {
@@ -115,7 +115,10 @@ export const redis: RateProvider = {
       requestedRates: out.requestedRates
     }
   },
-  updateRates: async (params: UpdateRatesParams): Promise<void> => {
+  updateRates: async (
+    params: UpdateRatesParams,
+    rightNow: Date
+  ): Promise<void> => {
     if (params.targetFiat !== 'USD') {
       return
     }
@@ -124,13 +127,13 @@ export const redis: RateProvider = {
       return
     }
 
-    const cryptoRateBuckets = groupCryptoRatesByTime(params.crypto)
+    const cryptoRateBuckets = groupCryptoRatesByTime(params.crypto, rightNow)
     for (const [date, cryptoRates] of cryptoRateBuckets.entries()) {
       const cryptoRedisKey = `rates_data:${date}:crypto`
       await hsetAsync(cryptoRedisKey, cryptoRates)
     }
 
-    const fiatRateBuckets = groupFiatRatesByTime(params.fiat)
+    const fiatRateBuckets = groupFiatRatesByTime(params.fiat, rightNow)
     for (const [date, fiatRates] of fiatRateBuckets.entries()) {
       const fiatRedisKey = `rates_data:${date}:fiat`
       await hsetAsync(fiatRedisKey, fiatRates)

src/v3/types.ts

@@ -102,7 +102,7 @@ export interface RateProvider {
     foundRates: FiatRateMap
     requestedRates: FiatRateMap
   }>
-  updateRates?: (params: UpdateRatesParams) => Promise<void>
+  updateRates?: (params: UpdateRatesParams, rightNow: Date) => Promise<void>
   engines?: Array<{
     frequency: Frequency
     engine: RateEngine

src/v3/utils.ts

@@ -110,6 +110,20 @@ export const createTokenId = (
   }
 }
 
+// Ignore requests with a future time with an allowance for clients that are close enough
+const isFutureTime = ({
+  rateDate,
+  rightNow,
+  fuzzFactor = 0
+}: {
+  rateDate: Date
+  rightNow: Date
+  fuzzFactor?: number
+}): boolean => {
+  const fuzzyFutureThreshold = new Date(rightNow.getTime() + fuzzFactor)
+  return rateDate > fuzzyFutureThreshold
+}
+
 // These functions reduce requested rates into buckets based on the date that
 // the providers can handle efficiently. The rates returned by the providers
 // can then be rematched with the requested rates
@@ -121,6 +135,16 @@ export const reduceRequestedCryptoRates = (
   const buckets: DateBuckets = new Map()
 
   requestedRates.forEach(rate => {
+    if (
+      isFutureTime({
+        rateDate: rate.isoDate,
+        rightNow,
+        fuzzFactor: ONE_MINUTE
+      })
+    ) {
+      return
+    }
+
     const rateTime = rate.isoDate.getTime()
 
     // Current rates (< five minutes old) use minute precision and historical rates use five minutes
@@ -193,11 +217,22 @@ export const expandReturnedCryptoRates = (
 }
 
 export const reduceRequestedFiatRates = (
-  requestedRates: FiatRateMap
+  requestedRates: FiatRateMap,
+  rightNow: Date
 ): DateBuckets => {
   const buckets: DateBuckets = new Map()
 
   requestedRates.forEach(rate => {
+    if (
+      isFutureTime({
+        rateDate: rate.isoDate,
+        rightNow,
+        fuzzFactor: ONE_MINUTE
+      })
+    ) {
+      return
+    }
+
     const rateTime = rate.isoDate.getTime()
 
     // Floor to the start of the interval bucket
@@ -243,14 +278,24 @@ export const expandReturnedFiatRates = (
 // This function breaks apart the requested rates into buckets of the given interval.
 type UpdateBuckets = Map<string, Record<string, number>>
 export const groupCryptoRatesByTime = (
-  requestedRates: CryptoRateMap
+  requestedRates: CryptoRateMap,
+  rightNow: Date
 ): UpdateBuckets => {
   const buckets: UpdateBuckets = new Map()
   const rightNowMs = new Date().getTime()
 
   requestedRates.forEach(cryptoRate => {
     if (cryptoRate.rate == null) return
 
+    if (
+      isFutureTime({
+        rateDate: cryptoRate.isoDate,
+        rightNow
+      })
+    ) {
+      return
+    }
+
     const rateTime = cryptoRate.isoDate.getTime()
 
     // Current rates (< five minutes old) use minute precision and historical rates use five minutes
@@ -269,13 +314,23 @@ export const groupCryptoRatesByTime = (
 }
 
 export const groupFiatRatesByTime = (
-  requestedRates: FiatRateMap
+  requestedRates: FiatRateMap,
+  rightNow: Date
 ): UpdateBuckets => {
   const buckets: UpdateBuckets = new Map()
 
   requestedRates.forEach(fiatRate => {
     if (fiatRate.rate == null) return
 
+    if (
+      isFutureTime({
+        rateDate: fiatRate.isoDate,
+        rightNow
+      })
+    ) {
+      return
+    }
+
     const rateTime = fiatRate.isoDate.getTime()
 
     // Floor to the start of the interval bucket