Skip to content

Commit c1dad33

Browse files
jannikluhnjannikluhn
authored
Merge pull request #472 from shutter-network/metrics
Add more metrics
2 parents 5f88611 + 327b898 commit c1dad33

File tree

11 files changed

+194
-12
lines changed

11 files changed

+194
-12
lines changed

‎rolling-shutter/keyperimpl/gnosis/database/gnosiskeyper.sqlc.gen.go

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rolling-shutter/keyperimpl/gnosis/database/sql/queries/gnosiskeyper.sql

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,7 @@ SELECT * FROM validator_registrations_synced_until LIMIT 1;
112112
SELECT nonce FROM validator_registrations
113113
WHERE validator_index = $1 AND block_number <= $2 AND tx_index <= $3 AND log_index <= $4
114114
ORDER BY block_number DESC, tx_index DESC, log_index DESC
115-
LIMIT 1;
115+
LIMIT 1;
116+
117+
-- name: GetNumValidatorRegistrations :one
118+
SELECT COUNT(*) FROM validator_registrations;

‎rolling-shutter/keyperimpl/gnosis/handlers.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package gnosis
33
import (
44
"context"
55
"database/sql"
6+
"fmt"
67
"math"
78

89
"github.com/jackc/pgx/v4"
@@ -290,5 +291,8 @@ func (h *DecryptionKeysHandler) HandleMessage(ctx context.Context, msg p2pmsg.Me
290291
if err != nil {
291292
return []p2pmsg.Message{}, errors.Wrap(err, "failed to set tx pointer")
292293
}
294+
eonString := fmt.Sprint(keys.Eon)
295+
metricsTxPointer.WithLabelValues(eonString).Set(float64(newTxPointer))
296+
metricsTxPointerAge.WithLabelValues(eonString).Set(0)
293297
return []p2pmsg.Message{}, nil
294298
}

‎rolling-shutter/keyperimpl/gnosis/messagingmiddleware.go

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"bytes"
55
"context"
66
"database/sql"
7+
"fmt"
8+
"time"
79

810
"github.com/jackc/pgx/v4"
911
"github.com/jackc/pgx/v4/pgxpool"
@@ -16,6 +18,7 @@ import (
1618
corekeyperdatabase "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/database"
1719
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyperimpl/gnosis/database"
1820
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyperimpl/gnosis/gnosisssztypes"
21+
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley"
1922
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/identitypreimage"
2023
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/retry"
2124
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
@@ -172,14 +175,21 @@ func (i *MessagingMiddleware) interceptDecryptionKeyShares(
172175
)
173176
}
174177

175-
msg := proto.Clone(originalMsg).(*p2pmsg.DecryptionKeyShares)
176-
msg.Extra = &p2pmsg.DecryptionKeyShares_Gnosis{
177-
Gnosis: &p2pmsg.GnosisDecryptionKeySharesExtra{
178-
Slot: uint64(currentDecryptionTrigger.Slot),
179-
TxPointer: uint64(currentDecryptionTrigger.TxPointer),
180-
Signature: signature,
181-
},
178+
extra := &p2pmsg.GnosisDecryptionKeySharesExtra{
179+
Slot: uint64(currentDecryptionTrigger.Slot),
180+
TxPointer: uint64(currentDecryptionTrigger.TxPointer),
181+
Signature: signature,
182182
}
183+
msg := proto.Clone(originalMsg).(*p2pmsg.DecryptionKeyShares)
184+
msg.Extra = &p2pmsg.DecryptionKeyShares_Gnosis{Gnosis: extra}
185+
slotStartTimestamp := medley.SlotToTimestamp(
186+
extra.Slot,
187+
i.config.Gnosis.GenesisSlotTimestamp,
188+
i.config.Gnosis.SecondsPerSlot,
189+
)
190+
slotStartTime := time.Unix(int64(slotStartTimestamp), 0)
191+
delta := time.Since(slotStartTime)
192+
metricsKeySharesSentTimeDelta.WithLabelValues(fmt.Sprint(originalMsg.Eon)).Observe(delta.Seconds())
183193
return msg, nil
184194
}
185195

@@ -266,6 +276,15 @@ func (i *MessagingMiddleware) interceptDecryptionKeys(
266276
Int("num-signatures", len(signaturesDB)).
267277
Int("num-keys", len(msg.Keys)).
268278
Msg("sending keys")
279+
280+
slotStartTimestamp := medley.SlotToTimestamp(
281+
extra.Slot,
282+
i.config.Gnosis.GenesisSlotTimestamp,
283+
i.config.Gnosis.SecondsPerSlot,
284+
)
285+
slotStartTime := time.Unix(int64(slotStartTimestamp), 0)
286+
delta := time.Since(slotStartTime)
287+
metricsKeysSentTimeDelta.WithLabelValues(fmt.Sprint(originalMsg.Eon)).Observe(delta.Seconds())
269288
return msg, nil
270289
}
271290

@@ -294,5 +313,8 @@ func (i *MessagingMiddleware) advanceTxPointer(ctx context.Context, msg *p2pmsg.
294313
if err != nil {
295314
return errors.Wrap(err, "failed to set tx pointer")
296315
}
316+
eonString := fmt.Sprint(msg.Eon)
317+
metricsTxPointer.WithLabelValues(eonString).Set(float64(newTxPointer))
318+
metricsTxPointerAge.WithLabelValues(eonString).Set(0)
297319
return nil
298320
}

‎rolling-shutter/keyperimpl/gnosis/metrics.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package gnosis
2+
3+
import "github.com/prometheus/client_golang/prometheus"
4+
5+
var metricsTxPointer = prometheus.NewGaugeVec(
6+
prometheus.GaugeOpts{
7+
Namespace: "shutter",
8+
Subsystem: "gnosis",
9+
Name: "tx_pointer",
10+
Help: "Current value of the tx pointer",
11+
},
12+
[]string{"eon"},
13+
)
14+
15+
var metricsTxPointerAge = prometheus.NewGaugeVec(
16+
prometheus.GaugeOpts{
17+
Namespace: "shutter",
18+
Subsystem: "gnosis",
19+
Name: "tx_pointer_age_blocks",
20+
Help: "Current age of the tx pointer",
21+
},
22+
[]string{"eon"},
23+
)
24+
25+
var metricsLatestTxSubmittedEventIndex = prometheus.NewGaugeVec(
26+
prometheus.GaugeOpts{
27+
Namespace: "shutter",
28+
Subsystem: "gnosis",
29+
Name: "latest_tx_submitted_event_index",
30+
Help: "Index of the latest TxSubmitted event",
31+
},
32+
[]string{"eon"},
33+
)
34+
35+
var metricsTxSubmittedEventsSyncedUntil = prometheus.NewGauge(
36+
prometheus.GaugeOpts{
37+
Namespace: "shutter",
38+
Subsystem: "gnosis",
39+
Name: "tx_submitted_events_synced_until",
40+
Help: "Block number until which TxSubmitted events have been synced",
41+
},
42+
)
43+
44+
var metricsValidatorRegistrationsSyncedUntil = prometheus.NewGauge(
45+
prometheus.GaugeOpts{
46+
Namespace: "shutter",
47+
Subsystem: "gnosis",
48+
Name: "validator_registrations_synced_until",
49+
Help: "Block number until which validator registration events have been synced",
50+
},
51+
)
52+
53+
var metricsNumValidatorRegistrations = prometheus.NewGauge(
54+
prometheus.GaugeOpts{
55+
Namespace: "shutter",
56+
Subsystem: "gnosis",
57+
Name: "num_validator_registrations_total",
58+
Help: "Number of synced validator registrations",
59+
},
60+
)
61+
62+
var slotTimeDeltaBuckets = []float64{-5, -4.5, -4.0, -3.5, -3.0, -2.5, -2.0, -1.5, -1.0, -0.5, -0, 1.0, 100}
63+
64+
var metricsKeysSentTimeDelta = prometheus.NewHistogramVec(
65+
prometheus.HistogramOpts{
66+
Namespace: "shutter",
67+
Subsystem: "gnosis",
68+
Name: "keys_sent_time_delta_seconds",
69+
Help: "Time at which keys are sent relative to slot",
70+
Buckets: slotTimeDeltaBuckets,
71+
},
72+
[]string{"eon"},
73+
)
74+
75+
var metricsKeySharesSentTimeDelta = prometheus.NewHistogramVec(
76+
prometheus.HistogramOpts{
77+
Namespace: "shutter",
78+
Subsystem: "gnosis",
79+
Name: "key_shares_sent_time_delta_seconds",
80+
Help: "Time at which key shares are sent relative to slot",
81+
Buckets: slotTimeDeltaBuckets,
82+
},
83+
[]string{"eon"},
84+
)
85+
86+
func init() {
87+
prometheus.MustRegister(metricsTxPointer)
88+
prometheus.MustRegister(metricsTxPointerAge)
89+
prometheus.MustRegister(metricsLatestTxSubmittedEventIndex)
90+
prometheus.MustRegister(metricsTxSubmittedEventsSyncedUntil)
91+
prometheus.MustRegister(metricsValidatorRegistrationsSyncedUntil)
92+
prometheus.MustRegister(metricsNumValidatorRegistrations)
93+
prometheus.MustRegister(metricsKeysSentTimeDelta)
94+
prometheus.MustRegister(metricsKeySharesSentTimeDelta)
95+
}

‎rolling-shutter/keyperimpl/gnosis/newslot.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ func getTxPointer(ctx context.Context, db *pgxpool.Pool, eon int64, maxTxPointer
163163
var txPointer int64
164164
var txPointerAge int64
165165
var txPointerOutdated bool
166+
eonString := fmt.Sprint(eon)
166167
txPointerDB, err := gnosisKeyperDB.GetTxPointer(ctx, eon)
167168
if err == pgx.ErrNoRows {
168169
log.Info().Int64("eon", eon).Msg("initializing tx pointer")
@@ -192,6 +193,7 @@ func getTxPointer(ctx context.Context, db *pgxpool.Pool, eon int64, maxTxPointer
192193
if txPointerDB.Age.Valid {
193194
txPointerOutdated = txPointerAge > maxTxPointerAge
194195
} else {
196+
txPointerAge = math.MaxInt64
195197
txPointerOutdated = true
196198
}
197199
}
@@ -214,6 +216,8 @@ func getTxPointer(ctx context.Context, db *pgxpool.Pool, eon int64, maxTxPointer
214216
return 0, errors.Wrap(err, "failed to query transaction submitted event count from db")
215217
}
216218
}
219+
metricsTxPointer.WithLabelValues(eonString).Set(float64(txPointer))
220+
metricsTxPointerAge.WithLabelValues(eonString).Set(float64(txPointerAge))
217221
return txPointer, nil
218222
}
219223

‎rolling-shutter/keyperimpl/gnosis/sequencersyncer.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ func (s *SequencerSyncer) syncRange(
9494
Int("num-inserted-events", len(filteredEvents)).
9595
Int("num-discarded-events", len(events)-len(filteredEvents)).
9696
Msg("synced sequencer contract")
97+
metricsTxSubmittedEventsSyncedUntil.Set(float64(end))
9798
return nil
9899
}
99100

@@ -178,6 +179,7 @@ func (s *SequencerSyncer) insertTransactionSubmittedEvents(
178179
if err != nil {
179180
return errors.Wrap(err, "failed to insert transaction submitted event into db")
180181
}
182+
metricsLatestTxSubmittedEventIndex.WithLabelValues(string(event.Eon)).Set(float64(nextEventIndex))
181183
nextEventIndices[event.Eon]++
182184
log.Debug().
183185
Int64("index", nextEventIndex).

‎rolling-shutter/keyperimpl/gnosis/validatorsyncer.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,19 @@ func (v *ValidatorSyncer) syncRange(ctx context.Context, start, end uint64) erro
9696
if err != nil {
9797
return err
9898
}
99+
numRegistrations, err := db.GetNumValidatorRegistrations(ctx)
100+
if err != nil {
101+
return errors.Wrap(err, "failed to get number of validator registrations")
102+
}
99103
log.Info().
100104
Uint64("start-block", start).
101105
Uint64("end-block", end).
102106
Int("num-inserted-events", len(filteredEvents)).
103107
Int("num-discarded-events", len(events)-len(filteredEvents)).
108+
Int64("num-registrations", numRegistrations).
104109
Msg("synced validator registry")
110+
metricsNumValidatorRegistrations.Set(float64(numRegistrations))
111+
metricsValidatorRegistrationsSyncedUntil.Set(float64(end))
105112
return nil
106113
}
107114

‎rolling-shutter/medley/slots.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,7 @@ func BlockTimestampToSlot(blockTimestamp uint64, genesisSlotTimestamp uint64, se
77
func SlotToEpoch(slot uint64, slotsPerEpoch uint64) uint64 {
88
return slot / slotsPerEpoch
99
}
10+
11+
func SlotToTimestamp(slot uint64, genesisSlotTimestamp uint64, secondsPerSlot uint64) uint64 {
12+
return genesisSlotTimestamp + slot*secondsPerSlot
13+
}

‎rolling-shutter/p2p/messaging.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,7 @@ func (r *ValidatorRegistry) GetCombinedValidator(topic string) pubsub.ValidatorE
3434
startTime := time.Now()
3535
defer func() {
3636
elapsedTime := time.Since(startTime)
37-
log.Debug().
38-
Str("topic", topic).
39-
Str("duration", elapsedTime.String()).
40-
Msg("validated message")
37+
metricsP2PMessageValidationTime.WithLabelValues(topic).Observe(elapsedTime.Seconds())
4138
}()
4239

4340
ignored := false
@@ -275,13 +272,16 @@ func (m *P2PMessaging) runHandleMessages(ctx context.Context) error {
275272
if !ok {
276273
return nil
277274
}
275+
startTime := time.Now()
278276
if err := m.handle(ctx, msg); err != nil {
279277
log.Info().
280278
Err(err).
281279
Str("topic", msg.GetTopic()).
282280
Str("sender-id", msg.GetFrom().String()).
283281
Msg("failed to handle message")
284282
}
283+
elapsedTime := time.Since(startTime)
284+
metricsP2PMessageHandlingTime.WithLabelValues(msg.GetTopic()).Observe(elapsedTime.Seconds())
285285
case <-ctx.Done():
286286
return ctx.Err()
287287
}

0 commit comments

Comments
 (0)