Skip to content

Commit da9df36

Browse files
authored
TXM - add initial prom metrics (#305)
* TXM - add initial prom metrics * update vendor hash * gomods tidy * lint * update vendor hash * lint
1 parent 5e041e6 commit da9df36

File tree

6 files changed

+178
-18
lines changed

6 files changed

+178
-18
lines changed

cmd/chainlink-ton/lock.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Notice: `pkgs.lib.fakeHash` can be used as a placeholder,
22
# but `nix-lock-tidy` will only replace actual hashes.
33
{pkgs}: {
4-
chainlink-ton = "sha256-Qp23zuGQmZrNGly0QyyXqMUBxvH5OwtNAg+TriPn3TQ=";
4+
chainlink-ton = "sha256-iuLL/v89uyAvTOvF2GyrQInB/8SQyskhhzdtOqJiISE=";
55
}

go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@ require (
1010
github.com/ethereum/go-ethereum v1.16.2
1111
github.com/gagliardetto/solana-go v1.13.0
1212
github.com/pelletier/go-toml/v2 v2.2.4
13+
github.com/prometheus/client_golang v1.23.0
1314
github.com/smartcontractkit/chain-selectors v1.0.72
1415
github.com/smartcontractkit/chainlink-ccip v0.1.1-solana.0.20251024142759-093ed1b4017f
1516
github.com/smartcontractkit/chainlink-common v0.9.6-0.20251003171904-99a82a53b142
1617
github.com/smartcontractkit/libocr v0.0.0-20250905115425-2785a5cee79d
1718
github.com/spf13/cobra v1.10.1
1819
github.com/stretchr/testify v1.11.1
1920
github.com/xssnick/tonutils-go v1.14.1
21+
go.opentelemetry.io/otel v1.37.0
22+
go.opentelemetry.io/otel/metric v1.37.0
2023
go.uber.org/zap v1.27.0
2124
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc
2225
golang.org/x/net v0.42.0
@@ -89,7 +92,6 @@ require (
8992
github.com/pelletier/go-toml v1.9.5 // indirect
9093
github.com/pkg/errors v0.9.1 // indirect
9194
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
92-
github.com/prometheus/client_golang v1.23.0 // indirect
9395
github.com/prometheus/client_model v0.6.2 // indirect
9496
github.com/prometheus/common v0.65.0 // indirect
9597
github.com/prometheus/procfs v0.16.1 // indirect
@@ -109,7 +111,6 @@ require (
109111
go.mongodb.org/mongo-driver v1.17.2 // indirect
110112
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
111113
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
112-
go.opentelemetry.io/otel v1.37.0 // indirect
113114
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 // indirect
114115
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 // indirect
115116
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 // indirect
@@ -121,7 +122,6 @@ require (
121122
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect
122123
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 // indirect
123124
go.opentelemetry.io/otel/log v0.13.0 // indirect
124-
go.opentelemetry.io/otel/metric v1.37.0 // indirect
125125
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
126126
go.opentelemetry.io/otel/sdk/log v0.13.0 // indirect
127127
go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect

integration-tests/txm/txm_test.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,18 +52,19 @@ func TestTxmLocal(t *testing.T) {
5252
config := txm.DefaultConfigSet
5353
config.ConfirmPollInterval = commonconfig.MustNewDuration(2 * time.Second)
5454

55-
runTxmTest(t, logger, config, tonChain, keystore, 5)
55+
runTxmTest(t, logger, config, tonChain, string(chainsel.TON_LOCALNET.ChainID), keystore, 5)
5656
}
5757

58-
func runTxmTest(t *testing.T, logger logger.Logger, config txm.Config, tonChain cldf_ton.Chain, keystore loop.Keystore, iterations int) {
58+
func runTxmTest(t *testing.T, logger logger.Logger, config txm.Config, tonChain cldf_ton.Chain, chainID string, keystore loop.Keystore, iterations int) {
5959
signedClientProvider := func(ctx context.Context) (tracetracking.SignedAPIClient, error) {
6060
return tracetracking.SignedAPIClient{
6161
Client: tonChain.Client,
6262
Wallet: *tonChain.Wallet,
6363
}, nil
6464
}
65-
tonTxm := txm.New(logger, keystore, signedClientProvider, config)
66-
err := tonTxm.Start(t.Context())
65+
tonTxm, err := txm.New(logger, chainID, keystore, signedClientProvider, config)
66+
require.NoError(t, err)
67+
err = tonTxm.Start(t.Context())
6768
require.NoError(t, err)
6869
defer func() {
6970
_ = tonTxm.Close()

pkg/relay/chain.go

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,14 +104,14 @@ func newChain(cfg *config.TOMLConfig, loopKs loop.Keystore, lggr logger.Logger,
104104

105105
// TODO(@jadepark-dev): TXM technically doesn't need SignedAPIClient, revisit to refactor
106106
signedClientProvider := commonutils.NewLazyLoadCtx(func(ctx context.Context) (tracetracking.SignedAPIClient, error) {
107-
tonClient, err := ch.GetClient(ctx)
108-
if err != nil {
109-
return tracetracking.SignedAPIClient{}, fmt.Errorf("failed to create TON client for chain ID %s: %w", cfg.ChainID, err)
107+
tonClient, err1 := ch.GetClient(ctx)
108+
if err1 != nil {
109+
return tracetracking.SignedAPIClient{}, fmt.Errorf("failed to create TON client for chain ID %s: %w", cfg.ChainID, err1)
110110
}
111111

112-
signerWallet, err := ch.GetSignerWallet(ctx, tonClient, loopKs, 0)
113-
if err != nil {
114-
return tracetracking.SignedAPIClient{}, fmt.Errorf("failed to get signer wallet for chain ID %s: %w", cfg.ChainID, err)
112+
signerWallet, err1 := ch.GetSignerWallet(ctx, tonClient, loopKs, 0)
113+
if err1 != nil {
114+
return tracetracking.SignedAPIClient{}, fmt.Errorf("failed to get signer wallet for chain ID %s: %w", cfg.ChainID, err1)
115115
}
116116

117117
return tracetracking.SignedAPIClient{
@@ -120,7 +120,10 @@ func newChain(cfg *config.TOMLConfig, loopKs loop.Keystore, lggr logger.Logger,
120120
}, nil
121121
})
122122

123-
ch.txm = txm.New(lggr, loopKs, signedClientProvider.Get, *ch.cfg.TxManager())
123+
ch.txm, err = txm.New(lggr, ch.id, loopKs, signedClientProvider.Get, *ch.cfg.TxManager())
124+
if err != nil {
125+
return nil, fmt.Errorf("failed to create TON TXM for chain ID %s: %w", cfg.ChainID, err)
126+
}
124127

125128
clientProvider := func(ctx context.Context) (ton.APIClientWrapped, error) {
126129
signedClient, err := signedClientProvider.Get(ctx)

pkg/txm/metrics.go

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package txm
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"github.com/prometheus/client_golang/prometheus"
8+
"github.com/prometheus/client_golang/prometheus/promauto"
9+
"go.opentelemetry.io/otel/attribute"
10+
"go.opentelemetry.io/otel/metric"
11+
12+
"github.com/smartcontractkit/chainlink-common/pkg/beholder"
13+
"github.com/smartcontractkit/chainlink-common/pkg/metrics"
14+
)
15+
16+
var (
17+
// Successful transactions
18+
promTonTxmSuccessTxs = promauto.NewCounterVec(prometheus.CounterOpts{
19+
Name: "ton_txm_tx_success",
20+
Help: "Number of finalized transactions that are included and successfully executed on chain",
21+
}, []string{"chainID"})
22+
promTonTxmFinalizedTxs = promauto.NewCounterVec(prometheus.CounterOpts{
23+
Name: "ton_txm_tx_finalized",
24+
Help: "Number of transactions that are finalized on chain. Can include both successful and reverted txs",
25+
}, []string{"chainID"})
26+
27+
// Inflight transactions
28+
promTonTxmPendingTxs = promauto.NewGaugeVec(prometheus.GaugeOpts{
29+
Name: "ton_txm_tx_pending",
30+
Help: "Number of transactions that are pending confirmation",
31+
}, []string{"chainID"})
32+
33+
// Error cases
34+
promTonTxmFailedToBroadcastTxs = promauto.NewCounterVec(prometheus.CounterOpts{
35+
Name: "ton_txm_tx_error_broadcast",
36+
Help: "Number of transactions that failed to be broadcasted even after all retries",
37+
}, []string{"chainID"})
38+
promTonTxmRevertTxs = promauto.NewCounterVec(prometheus.CounterOpts{
39+
Name: "ton_txm_tx_error_revert",
40+
Help: "Number of finalized transactions that are included and failed onchain",
41+
}, []string{"chainID"})
42+
)
43+
44+
type tonTxmMetrics struct {
45+
metrics.Labeler
46+
chainID string
47+
48+
// successful transactions
49+
successTxs metric.Int64Counter
50+
finalizedTxs metric.Int64Counter
51+
52+
// inflight transactions
53+
pendingTxs metric.Int64Gauge
54+
55+
// error cases
56+
failedToBroadcastTxs metric.Int64Counter
57+
revertTxs metric.Int64Counter
58+
}
59+
60+
func newTonTxmMetrics(chainID string) (*tonTxmMetrics, error) {
61+
m := beholder.GetMeter()
62+
var err error
63+
64+
successTxs, err := m.Int64Counter("ton_txm_tx_success")
65+
if err != nil {
66+
return nil, fmt.Errorf("failed to register ton success txs: %w", err)
67+
}
68+
69+
finalizedTxs, err := m.Int64Counter("ton_txm_tx_finalized")
70+
if err != nil {
71+
return nil, fmt.Errorf("failed to register ton finalized txs: %w", err)
72+
}
73+
74+
pendingTxs, err := m.Int64Gauge("ton_txm_tx_pending")
75+
if err != nil {
76+
return nil, fmt.Errorf("failed to register ton pending txs: %w", err)
77+
}
78+
79+
failedToBroadcastTxs, err := m.Int64Counter("ton_txm_tx_error_broadcast")
80+
if err != nil {
81+
return nil, fmt.Errorf("failed to register ton failed to broadcast txs: %w", err)
82+
}
83+
84+
revertTxs, err := m.Int64Counter("ton_txm_tx_error_revert")
85+
if err != nil {
86+
return nil, fmt.Errorf("failed to register ton revert txs: %w", err)
87+
}
88+
89+
return &tonTxmMetrics{
90+
chainID: chainID,
91+
Labeler: metrics.NewLabeler().With("chainID", chainID),
92+
93+
successTxs: successTxs,
94+
finalizedTxs: finalizedTxs,
95+
pendingTxs: pendingTxs,
96+
97+
failedToBroadcastTxs: failedToBroadcastTxs,
98+
revertTxs: revertTxs,
99+
}, nil
100+
}
101+
102+
func (m *tonTxmMetrics) GetOtelAttributes() []attribute.KeyValue {
103+
return beholder.OtelAttributes(m.Labels).AsStringAttributes()
104+
}
105+
106+
func (m *tonTxmMetrics) IncrementSuccessTxs(ctx context.Context) {
107+
promTonTxmSuccessTxs.WithLabelValues(m.chainID).Add(1)
108+
m.successTxs.Add(ctx, 1, metric.WithAttributes(m.GetOtelAttributes()...))
109+
}
110+
111+
func (m *tonTxmMetrics) IncrementFinalizedTxs(ctx context.Context) {
112+
promTonTxmFinalizedTxs.WithLabelValues(m.chainID).Add(1)
113+
m.finalizedTxs.Add(ctx, 1, metric.WithAttributes(m.GetOtelAttributes()...))
114+
}
115+
116+
func (m *tonTxmMetrics) SetPendingTxs(ctx context.Context, count int) {
117+
promTonTxmPendingTxs.WithLabelValues(m.chainID).Set(float64(count))
118+
m.pendingTxs.Record(ctx, int64(count), metric.WithAttributes(m.GetOtelAttributes()...))
119+
}
120+
121+
func (m *tonTxmMetrics) IncrementFailedToBroadcastTxs(ctx context.Context) {
122+
promTonTxmFailedToBroadcastTxs.WithLabelValues(m.chainID).Add(1)
123+
m.failedToBroadcastTxs.Add(ctx, 1, metric.WithAttributes(m.GetOtelAttributes()...))
124+
}
125+
126+
func (m *tonTxmMetrics) IncrementRevertTxs(ctx context.Context) {
127+
promTonTxmRevertTxs.WithLabelValues(m.chainID).Add(1)
128+
m.revertTxs.Add(ctx, 1, metric.WithAttributes(m.GetOtelAttributes()...))
129+
}

pkg/txm/txm.go

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ type Txm struct {
3939
logger logger.Logger
4040
keystore loop.Keystore
4141
config Config
42+
chainID string
43+
metrics *tonTxmMetrics
4244

4345
clientProvider func(context.Context) (tracetracking.SignedAPIClient, error)
4446
broadcastChan chan *Tx
@@ -59,18 +61,31 @@ type Request struct {
5961
ID *string // Optional: unique ID for transaction tracking
6062
}
6163

62-
func New(lgr logger.Logger, keystore loop.Keystore, clientProvider func(context.Context) (tracetracking.SignedAPIClient, error), config Config) *Txm {
64+
func New(
65+
lggr logger.Logger,
66+
chainID string,
67+
keystore loop.Keystore,
68+
clientProvider func(context.Context) (tracetracking.SignedAPIClient, error),
69+
config Config,
70+
) (*Txm, error) {
71+
metrics, err := newTonTxmMetrics(chainID)
72+
if err != nil {
73+
return nil, fmt.Errorf("failed to initialize metrics: %w", err)
74+
}
75+
6376
txm := &Txm{
64-
logger: logger.Named(lgr, "Txm"),
77+
logger: logger.Named(lggr, "Txm"),
6578
keystore: keystore,
6679
config: config,
80+
chainID: chainID,
81+
metrics: metrics,
6782
clientProvider: clientProvider,
6883
broadcastChan: make(chan *Tx, config.BroadcastChanSize),
6984
accountStore: NewAccountStore(),
7085
stop: make(chan struct{}),
7186
}
7287

73-
return txm
88+
return txm, nil
7489
}
7590

7691
func (t *Txm) Name() string {
@@ -263,6 +278,7 @@ func (t *Txm) broadcastWithRetry(ctx context.Context, tx *Tx, msg *wallet.Messag
263278
}
264279

265280
if err != nil {
281+
t.metrics.IncrementFailedToBroadcastTxs(ctx)
266282
t.logger.Errorw("failed to broadcast tx after retries",
267283
"txID", txID,
268284
"err", err,
@@ -367,13 +383,24 @@ func (t *Txm) checkUnconfirmed(ctx context.Context) {
367383
continue
368384
}
369385

386+
t.metrics.IncrementFinalizedTxs(ctx)
370387
if traceSucceeded {
388+
t.metrics.IncrementSuccessTxs(ctx)
371389
t.logger.Infow("transaction confirmed", "LT", unconfirmedTx.LT, "exitCode", exitCode)
372390
} else {
391+
t.metrics.IncrementRevertTxs(ctx)
373392
t.logger.Warnw("transaction failed", "LT", unconfirmedTx.LT, "exitCode", exitCode)
374393
}
375394
}
376395
}
396+
397+
// Update pending transactions metric after processing
398+
allUnconfirmedTxsByAccounts := t.accountStore.GetAllUnconfirmed()
399+
totalPending := 0
400+
for _, unconfirmedTxs := range allUnconfirmedTxsByAccounts {
401+
totalPending += len(unconfirmedTxs)
402+
}
403+
t.metrics.SetPendingTxs(ctx, totalPending)
377404
}
378405

379406
// Periodically cleans up finalized and expired transactions from the TxStore.

0 commit comments

Comments
 (0)