@@ -179,45 +179,56 @@ pub async fn run_keeper_threads(
179179 } ;
180180
181181 loop {
182- // There isn't a loop for indefinite trials. There is a new thread being spawned every `TRACK_INTERVAL` seconds.
183- // If rpc start fails all of these threads will just exit, instead of retrying.
184- // We are tracking rpc failures elsewhere, so it's fine.
185- spawn (
186- track_provider (
187- chain_id. clone ( ) ,
188- contract. clone ( ) ,
189- provider_address,
190- keeper_metrics. clone ( ) ,
191- )
192- . in_current_span ( ) ,
193- ) ;
194- spawn (
195- track_balance (
196- chain_id. clone ( ) ,
197- contract. client ( ) ,
198- keeper_address,
199- keeper_metrics. clone ( ) ,
200- )
201- . in_current_span ( ) ,
202- ) ;
203- spawn (
204- track_accrued_pyth_fees (
205- chain_id. clone ( ) ,
206- contract. clone ( ) ,
207- keeper_metrics. clone ( ) ,
208- )
209- . in_current_span ( ) ,
210- ) ;
211- spawn (
212- track_block_timestamp_lag (
213- chain_id. clone ( ) ,
214- contract. client ( ) ,
215- keeper_metrics. clone ( ) ,
216- )
217- . in_current_span ( ) ,
218- ) ;
219-
220182 time:: sleep ( TRACK_INTERVAL ) . await ;
183+
184+ // Track provider info and balance sequentially. Note that the tracking is done sequentially with the
185+ // timestamp last. If there is a persistent error in any of these methods, the timestamp will lag behind
186+ // current time and trigger an alert.
187+ if let Err ( e) = track_provider (
188+ chain_id. clone ( ) ,
189+ contract. clone ( ) ,
190+ provider_address,
191+ keeper_metrics. clone ( ) ,
192+ )
193+ . await
194+ {
195+ tracing:: error!( "Error tracking provider: {:?}" , e) ;
196+ continue ;
197+ }
198+
199+ if let Err ( e) = track_balance (
200+ chain_id. clone ( ) ,
201+ contract. client ( ) ,
202+ keeper_address,
203+ keeper_metrics. clone ( ) ,
204+ )
205+ . await
206+ {
207+ tracing:: error!( "Error tracking balance: {:?}" , e) ;
208+ continue ;
209+ }
210+
211+ if let Err ( e) = track_accrued_pyth_fees (
212+ chain_id. clone ( ) ,
213+ contract. clone ( ) ,
214+ keeper_metrics. clone ( ) ,
215+ )
216+ . await
217+ {
218+ tracing:: error!( "Error tracking accrued pyth fees: {:?}" , e) ;
219+ continue ;
220+ }
221+
222+ if let Err ( e) = track_block_timestamp_lag (
223+ chain_id. clone ( ) ,
224+ contract. client ( ) ,
225+ keeper_metrics. clone ( ) ,
226+ )
227+ . await
228+ {
229+ tracing:: error!( "Error tracking block timestamp lag: {:?}" , e) ;
230+ continue ;
231+ }
221232 }
222233 }
223234 . in_current_span ( ) ,
0 commit comments