Skip to content

Commit 50fa277

Browse files
authored
f(ILO): improve perf price-oracle (#47)
* f(ILO): improve perf price-oracle * f(ILO): update metrics
1 parent 2c0eb46 commit 50fa277

File tree

3 files changed

+51
-32
lines changed

3 files changed

+51
-32
lines changed

go.mod

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.24.0
44

55
require (
66
cosmossdk.io/math v1.4.0
7-
github.com/InjectiveLabs/metrics v0.0.10
7+
github.com/InjectiveLabs/metrics v0.0.21
88
github.com/InjectiveLabs/sdk-go v1.58.0
99
github.com/InjectiveLabs/suplog v1.3.4
1010
github.com/cometbft/cometbft v1.0.1
@@ -56,6 +56,7 @@ require (
5656
github.com/DataDog/datadog-go/v5 v5.3.0 // indirect
5757
github.com/DataDog/go-libddwaf/v2 v2.3.2 // indirect
5858
github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
59+
github.com/DataDog/gostackparse v0.7.0 // indirect
5960
github.com/DataDog/sketches-go v1.4.2 // indirect
6061
github.com/DataDog/zstd v1.5.6 // indirect
6162
github.com/Microsoft/go-winio v0.6.2 // indirect
@@ -133,6 +134,7 @@ require (
133134
github.com/google/btree v1.1.3 // indirect
134135
github.com/google/flatbuffers v25.2.10+incompatible // indirect
135136
github.com/google/go-cmp v0.7.0 // indirect
137+
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
136138
github.com/google/uuid v1.6.0 // indirect
137139
github.com/gorilla/handlers v1.5.2 // indirect
138140
github.com/gorilla/mux v1.8.1 // indirect
@@ -168,6 +170,7 @@ require (
168170
github.com/mattn/go-isatty v0.0.20 // indirect
169171
github.com/mattn/go-runewidth v0.0.16 // indirect
170172
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
173+
github.com/mixpanel/mixpanel-go v1.2.1 // indirect
171174
github.com/mmcloughlin/addchain v0.4.0 // indirect
172175
github.com/mtibben/percent v0.2.1 // indirect
173176
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -185,6 +188,7 @@ require (
185188
github.com/prometheus/common v0.62.0 // indirect
186189
github.com/prometheus/procfs v0.15.1 // indirect
187190
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
191+
github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect
188192
github.com/rivo/uniseg v0.4.7 // indirect
189193
github.com/rogpeppe/go-internal v1.14.1 // indirect
190194
github.com/rs/zerolog v1.33.0 // indirect
@@ -197,6 +201,7 @@ require (
197201
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
198202
github.com/sirupsen/logrus v1.9.3 // indirect
199203
github.com/sourcegraph/conc v0.3.0 // indirect
204+
github.com/spaolacci/murmur3 v1.1.0 // indirect
200205
github.com/spf13/afero v1.12.0 // indirect
201206
github.com/spf13/cast v1.7.1 // indirect
202207
github.com/spf13/cobra v1.9.1 // indirect

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,6 +1510,8 @@ github.com/InjectiveLabs/ibc-go/v8 v8.7.0-evm-comet1-inj h1:yuSodqSSgIwSQyLUJ45j
15101510
github.com/InjectiveLabs/ibc-go/v8 v8.7.0-evm-comet1-inj/go.mod h1:JnZjgZ9KyhXp3HFjtGXkM2ByyPfJWjAnz2KMCmFK3po=
15111511
github.com/InjectiveLabs/metrics v0.0.10 h1:BoOwXnCtRRIPmq06jcI20pXZYE758eusaCI5jDOoN4U=
15121512
github.com/InjectiveLabs/metrics v0.0.10/go.mod h1:eYu++0DVUjk/jjV9WgvCo8gQU+16Yoyhp1iu+ghKNME=
1513+
github.com/InjectiveLabs/metrics v0.0.21 h1:/oEOJryuY+34+6pIYO+EvJVn7MV267OD1v9MkyM2Cpw=
1514+
github.com/InjectiveLabs/metrics v0.0.21/go.mod h1:oDwGiKYrI9EplQDJzzBwvKsdCNyvGeTonxb91HauZmk=
15131515
github.com/InjectiveLabs/sdk-go v1.58.0 h1:n5bPcZMYP1o4uJbvtqsknqvhu8Ff0l8f72D1jdA99ig=
15141516
github.com/InjectiveLabs/sdk-go v1.58.0/go.mod h1:/LNZhVsHMpb3YUnWcLu7+p3fUUz9XLxhEzFfebFR9Mg=
15151517
github.com/InjectiveLabs/suplog v1.3.4 h1:Ex3qj19FkKjlJVOpqEzLnPFlFtuV3FV4zR5w9JnG2/0=
@@ -3307,6 +3309,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
33073309
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
33083310
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
33093311
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
3312+
github.com/mixpanel/mixpanel-go v1.2.1 h1:iykbHKomTJjVoWU95Vt1sjZy4HLt8UOYacMEEEMFBok=
3313+
github.com/mixpanel/mixpanel-go v1.2.1/go.mod h1:mPGaNhBoZMJuLu8k7Y1KhU5n8Vw13rxQZZjHj+b9RLk=
33103314
github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
33113315
github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
33123316
github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=

internal/service/oracle/service.go

Lines changed: 41 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,7 @@ type oracleSvc struct {
5858
}
5959

6060
const (
61-
maxRespTime = 15 * time.Second
62-
maxRespHeadersTime = 15 * time.Second
63-
maxRespBytes = 10 * 1024 * 1024
64-
maxTxStatusRetries = 3
61+
maxRespTime = 3 * time.Second
6562
maxRetriesPerInterval = 3
6663
MaxRetriesReConnectWebSocket = 5
6764
)
@@ -195,7 +192,7 @@ func (s *oracleSvc) Start(ctx context.Context) (err error) {
195192
for ticker, pricePuller := range s.pricePullers {
196193
switch pricePuller.Provider() {
197194
case FeedProviderBinance, FeedProviderStork, FeedProviderDynamic:
198-
go s.processSetPriceFeed(ticker, pricePuller, dataC)
195+
go s.processSetPriceFeed(ctx, ticker, pricePuller, dataC)
199196
default:
200197
s.logger.WithField("provider", pricePuller.Provider()).Warningln("unsupported price feed provider")
201198
}
@@ -207,7 +204,7 @@ func (s *oracleSvc) Start(ctx context.Context) (err error) {
207204
return
208205
}
209206

210-
func (s *oracleSvc) processSetPriceFeed(ticker string, pricePuller PricePuller, dataC chan<- *PriceData) {
207+
func (s *oracleSvc) processSetPriceFeed(ctx context.Context, ticker string, pricePuller PricePuller, dataC chan<- *PriceData) {
211208
feedLogger := s.logger.WithFields(log.Fields{
212209
"ticker": ticker,
213210
"provider": pricePuller.ProviderName(),
@@ -216,36 +213,38 @@ func (s *oracleSvc) processSetPriceFeed(ticker string, pricePuller PricePuller,
216213
symbol := pricePuller.Symbol()
217214

218215
t := time.NewTimer(5 * time.Second)
216+
defer t.Stop()
217+
219218
for {
220219
select {
220+
case <-ctx.Done():
221+
feedLogger.Infoln("context cancelled, stopping price feed")
222+
return
221223
case <-t.C:
222-
ctx, cancelFn := context.WithTimeout(context.Background(), maxRespTime)
223-
defer cancelFn()
224+
var result *PriceData
225+
var err error
224226

225-
result, err := pricePuller.PullPrice(ctx)
227+
for i := 0; i < maxRetriesPerInterval; i++ {
228+
requestCtx, cancelFn := context.WithTimeout(ctx, maxRespTime)
229+
result, err = pricePuller.PullPrice(requestCtx)
230+
cancelFn()
226231

227-
if err != nil {
228-
metrics.ReportFuncError(s.svcTags)
229-
feedLogger.WithError(err).Warningln("retrying PullPrice after error")
230-
231-
for i := 0; i < maxRetriesPerInterval; i++ {
232-
if result, err = pricePuller.PullPrice(ctx); err != nil {
233-
time.Sleep(time.Second)
234-
continue
235-
}
232+
if err == nil {
236233
break
237234
}
238235

239-
if err != nil {
240-
metrics.ReportFuncCallAndTimingWithErr(s.svcTags)(&err)
241-
feedLogger.WithFields(log.Fields{
242-
"symbol": symbol,
243-
"retries": maxRetriesPerInterval,
244-
}).WithError(err).Errorln("failed to fetch price")
236+
time.Sleep(100 * time.Millisecond)
237+
}
245238

246-
t.Reset(pricePuller.Interval())
247-
continue
248-
}
239+
if err != nil {
240+
metrics.ReportFuncCallAndTimingWithErr(s.svcTags)(&err)
241+
feedLogger.WithFields(log.Fields{
242+
"symbol": symbol,
243+
"retries": maxRetriesPerInterval,
244+
}).WithError(err).Errorln("failed to fetch price")
245+
246+
t.Reset(pricePuller.Interval())
247+
continue
249248
}
250249

251250
if result != nil {
@@ -259,9 +258,9 @@ func (s *oracleSvc) processSetPriceFeed(ticker string, pricePuller PricePuller,
259258

260259
const (
261260
commitPriceBatchTimeLimit = 5 * time.Second
261+
chainMaxTimeLimit = 3 * time.Second
262262
commitPriceBatchSizeLimit = 100
263-
maxRetries = 6
264-
chainMaxTimeLimit = 5 * time.Second
263+
maxRetries = 3
265264
)
266265

267266
var pullIntervalChain = 500 * time.Millisecond
@@ -362,6 +361,8 @@ func (s *oracleSvc) commitSetPrices(ctx context.Context, dataC <-chan *PriceData
362361
defer doneFn()
363362

364363
expirationTimer := time.NewTimer(commitPriceBatchTimeLimit)
364+
defer expirationTimer.Stop()
365+
365366
pricesBatch := make(map[string]*PriceData)
366367
pricesMeta := make(map[string]int)
367368

@@ -409,6 +410,11 @@ func (s *oracleSvc) commitSetPrices(ctx context.Context, dataC <-chan *PriceData
409410

410411
for {
411412
select {
413+
case <-ctx.Done():
414+
s.logger.Infoln("context cancelled, stopping commitSetPrices")
415+
prevBatch, prevMeta := resetBatch()
416+
submitBatch(prevBatch, prevMeta, false)
417+
return
412418
case priceData, ok := <-dataC:
413419
if !ok {
414420
s.logger.Infoln("stopping committing prices")
@@ -485,12 +491,16 @@ func (s *oracleSvc) broadcastToClient(
485491
}, s.svcTags)
486492
}
487493

494+
diff := time.Since(ts)
495+
488496
batchLog.WithFields(log.Fields{
489497
"cosmosClient": cosmosClient.ClientContext().From,
490498
"height": txResp.TxResponse.Height,
491499
"hash": txResp.TxResponse.TxHash,
492-
"duration": time.Since(ts),
493-
}).Infoln("sent Tx successfully in ", time.Since(ts))
500+
"duration": diff,
501+
}).Infoln("sent Tx successfully in ", diff)
502+
503+
metrics.Timer("price_oracle.execution_time", diff, s.svcTags)
494504
return true
495505
}
496506

0 commit comments

Comments
 (0)