Skip to content

Commit bb7053d

Browse files
peachbitspeachbits
authored
Merge pull request #121 from EdgeApp/clean-rates-script
Add script to clean up rates documents from a provider
2 parents fa351d3 + 9f29ec0 commit bb7053d

File tree

2 files changed

+228
-0
lines changed

2 files changed

+228
-0
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## Unreleased
44

5+
- added: Add script to wipe out provider rates from docs
6+
57
## 3.1.0 (2025-10-25)
68

79
- added: v2 api wrapper around v3 endpoints

‎bin/clean-rates-docs.ts‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
import { asMaybe, uncleaner } from 'cleaners'
2+
import { asCouchDoc } from 'edge-server-tools'
3+
4+
import { dbData, dbSettings } from '../src/v3/providers/couch'
5+
import { asRateDocument, asTokenMap } from '../src/v3/types'
6+
7+
/*
8+
This script will scrub the rates documents of any rate mapped in the uid document
9+
*/
10+
11+
const mappingDocId = 'coingecko:automated'
12+
const endTime = 1757289600000 // rates server PR merged september 8th 2025
13+
const intervalMs = 1000 * 60 * 5 // five minutes
14+
15+
const isDebug = process.env.DEBUG === 'true' || process.env.DEBUG === '1'
16+
const batchSize = isDebug ? 3 : 100
17+
18+
const asRatesDoc = asCouchDoc(asRateDocument)
19+
const wasRatesDoc = uncleaner(asRatesDoc)
20+
21+
const main = async (): Promise<void> => {
22+
try {
23+
// Fetch mapping document
24+
const mappingDoc = await dbSettings.get(mappingDocId)
25+
const tokenMap = asCouchDoc(asTokenMap)(mappingDoc).doc
26+
const mappingKeys = new Set(Object.keys(tokenMap))
27+
28+
console.log(`Found ${mappingKeys.size} keys in ${mappingDocId} document`)
29+
30+
if (isDebug) {
31+
console.log(
32+
`DEBUG mode: Will collect ${batchSize} modified documents and print them, then exit without saving`
33+
)
34+
}
35+
36+
let docTime = Math.floor(new Date().getTime() / intervalMs) * intervalMs
37+
const batch: Array<{ id: string; rev?: string; doc: any }> = []
38+
const debugInfoMap = new Map<
39+
string,
40+
{
41+
before: Record<string, { USD: number }>
42+
after: Record<string, { USD: number }>
43+
removedKeys: string[]
44+
}
45+
>()
46+
let processedCount = 0
47+
let modifiedCount = 0
48+
49+
while (docTime > endTime) {
50+
const docId = new Date(docTime).toISOString()
51+
52+
try {
53+
// Fetch document from rates_data database
54+
const rawDoc = await dbData.get(docId)
55+
const rateDoc = asMaybe(asRatesDoc)(rawDoc)
56+
57+
if (rateDoc != null) {
58+
// Filter out keys that exist in the automated document
59+
const originalCryptoKeys = Object.keys(rateDoc.doc.crypto)
60+
const filteredCrypto: Record<string, { USD: number }> = {}
61+
const removedKeys: string[] = []
62+
63+
for (const key of originalCryptoKeys) {
64+
if (!mappingKeys.has(key)) {
65+
filteredCrypto[key] = rateDoc.doc.crypto[key]
66+
} else {
67+
removedKeys.push(key)
68+
}
69+
}
70+
71+
// Only add to batch if crypto object was modified
72+
if (
73+
originalCryptoKeys.length !== Object.keys(filteredCrypto).length
74+
) {
75+
batch.push({
76+
id: rateDoc.id,
77+
rev: rateDoc.rev,
78+
doc: {
79+
crypto: filteredCrypto,
80+
fiat: rateDoc.doc.fiat
81+
}
82+
})
83+
84+
// Store debug info separately if in debug mode
85+
if (isDebug) {
86+
debugInfoMap.set(rateDoc.id, {
87+
before: rateDoc.doc.crypto,
88+
after: filteredCrypto,
89+
removedKeys
90+
})
91+
}
92+
93+
modifiedCount++
94+
95+
// When batch reaches batchSize
96+
if (batch.length >= batchSize) {
97+
if (isDebug) {
98+
// Debug mode: print all documents in batch and exit
99+
console.log(
100+
`\n=== DEBUG: Batch of ${batch.length} Modified Documents ===\n`
101+
)
102+
for (let i = 0; i < batch.length; i++) {
103+
const item = batch[i]
104+
const debugInfo = debugInfoMap.get(item.id)
105+
if (debugInfo == null) continue
106+
107+
console.log(`\n--- Document ${i + 1}/${batch.length} ---`)
108+
console.log(`Document ID: ${item.id}`)
109+
console.log('\n--- BEFORE ---')
110+
console.log(JSON.stringify(debugInfo.before, null, 2))
111+
console.log('\n--- AFTER ---')
112+
console.log(JSON.stringify(debugInfo.after, null, 2))
113+
console.log('\n--- REMOVED KEYS ---')
114+
console.log(JSON.stringify(debugInfo.removedKeys, null, 2))
115+
console.log(
116+
`Total keys before: ${Object.keys(debugInfo.before).length}`
117+
)
118+
console.log(
119+
`Total keys after: ${Object.keys(debugInfo.after).length}`
120+
)
121+
console.log(`Keys removed: ${debugInfo.removedKeys.length}`)
122+
if (i < batch.length - 1) {
123+
console.log('\n' + '-'.repeat(50))
124+
}
125+
}
126+
console.log('\n=====================================')
127+
console.log('\n--- ALL CHANGED DOCUMENT IDs ---')
128+
console.log(
129+
JSON.stringify(batch.map(item => item.id).sort(), null, 2)
130+
)
131+
console.log(`\nTotal changed documents: ${batch.length}`)
132+
console.log('\n=====================================')
133+
console.log(
134+
`DEBUG mode: Exiting after printing ${batch.length} documents - no changes saved`
135+
)
136+
return
137+
} else {
138+
// Normal mode: save to database
139+
await dbData.bulk({
140+
docs: batch.map(wasRatesDoc)
141+
})
142+
processedCount += batch.length
143+
console.log(
144+
`Processed ${processedCount} documents, ${modifiedCount} modified`
145+
)
146+
batch.length = 0 // Clear batch
147+
}
148+
}
149+
}
150+
}
151+
} catch (error: unknown) {
152+
// Skip missing documents silently
153+
if (
154+
error != null &&
155+
typeof error === 'object' &&
156+
'statusCode' in error &&
157+
error.statusCode !== 404
158+
) {
159+
console.error(`Error processing document ${docId}:`, error)
160+
}
161+
}
162+
163+
docTime = docTime - intervalMs
164+
}
165+
166+
// Save any remaining documents in the batch (only in non-debug mode)
167+
if (batch.length > 0 && !isDebug) {
168+
await dbData.bulk({
169+
docs: batch.map(wasRatesDoc)
170+
})
171+
processedCount += batch.length
172+
} else if (batch.length > 0 && isDebug) {
173+
// In debug mode, print remaining documents if any
174+
console.log(
175+
`\n=== DEBUG: Remaining ${batch.length} Modified Documents ===\n`
176+
)
177+
for (let i = 0; i < batch.length; i++) {
178+
const item = batch[i]
179+
const debugInfo = debugInfoMap.get(item.id)
180+
if (debugInfo == null) continue
181+
182+
console.log(`\n--- Document ${i + 1}/${batch.length} ---`)
183+
console.log(`Document ID: ${item.id}`)
184+
console.log('\n--- BEFORE ---')
185+
console.log(JSON.stringify(debugInfo.before, null, 2))
186+
console.log('\n--- AFTER ---')
187+
console.log(JSON.stringify(debugInfo.after, null, 2))
188+
console.log('\n--- REMOVED KEYS ---')
189+
console.log(JSON.stringify(debugInfo.removedKeys, null, 2))
190+
console.log(
191+
`Total keys before: ${Object.keys(debugInfo.before).length}`
192+
)
193+
console.log(`Total keys after: ${Object.keys(debugInfo.after).length}`)
194+
console.log(`Keys removed: ${debugInfo.removedKeys.length}`)
195+
if (i < batch.length - 1) {
196+
console.log('\n' + '-'.repeat(50))
197+
}
198+
}
199+
console.log('\n=====================================')
200+
console.log('\n--- ALL CHANGED DOCUMENT IDs ---')
201+
console.log(JSON.stringify(batch.map(item => item.id).sort(), null, 2))
202+
console.log(`\nTotal changed documents: ${batch.length}`)
203+
console.log('\n=====================================')
204+
console.log(
205+
`DEBUG mode: Exiting after printing ${batch.length} documents - no changes saved`
206+
)
207+
}
208+
209+
console.log(
210+
`Completed: Processed ${processedCount} documents, ${modifiedCount} modified`
211+
)
212+
} catch (error: unknown) {
213+
console.error('Fatal error:', error)
214+
process.exit(1)
215+
}
216+
}
217+
218+
main()
219+
.then(() => {
220+
console.log('Script completed successfully')
221+
process.exit(0)
222+
})
223+
.catch((error: unknown) => {
224+
console.error('Script failed:', error)
225+
process.exit(1)
226+
})

0 commit comments

Comments
 (0)