Skip to content

Commit af0a041

Browse files
FricounetFricounet
authored
[metrics] Collect cache metrics (#706)
Add a prometheus collector to collect cache metrics exposed by each nydusd daemon. This will bring a better observability into the cache performances and the prefetch happening in the background. Metrics desccriptions are based on my understanding of nydusd + the discussions that happened in dragonflyoss/nydus#1803 Signed-off-by: Baptiste Girard-Carrabin <baptiste.girardcarrabin@datadoghq.com>
1 parent f81b919 commit af0a041

10 files changed

Lines changed: 242 additions & 11 deletions

File tree

‎pkg/daemon/types/types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,12 @@ type CacheMetrics struct {
9393
PrefetchDataAmount uint64 `json:"prefetch_data_amount"`
9494
PrefetchRequestsCount uint64 `json:"prefetch_requests_count"`
9595
PrefetchWorkers uint `json:"prefetch_workers"`
96+
PrefetchUnmergedChunks uint64 `json:"prefetch_unmerged_chunks"`
9697
PrefetchCumulativeTimeMillis uint64 `json:"prefetch_cumulative_time_millis"`
9798
PrefetchBeginTimeSecs uint64 `json:"prefetch_begin_time_secs"`
99+
PrefetchBeginTimeMillis uint64 `json:"prefetch_begin_time_millis"`
98100
PrefetchEndTimeSecs uint64 `json:"prefetch_end_time_secs"`
101+
PrefetchEndTimeMillis uint64 `json:"prefetch_end_time_millis"`
99102
BufferedBackendSize uint64 `json:"buffered_backend_size"`
103+
DataAllReady bool `json:"data_all_ready"`
100104
}

‎pkg/metrics/collector/cache.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright (c) 2025. Nydus Developers. All rights reserved.
3+
*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package collector
8+
9+
import (
10+
"github.com/containerd/log"
11+
"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
12+
"github.com/containerd/nydus-snapshotter/pkg/metrics/data"
13+
)
14+
15+
type CacheMetricsCollector struct {
16+
Metrics *types.CacheMetrics
17+
ImageRef string
18+
DaemonID string
19+
}
20+
21+
type CacheMetricsVecCollector struct {
22+
MetricsVec []CacheMetricsCollector
23+
}
24+
25+
func (c *CacheMetricsCollector) Collect() {
26+
if c.Metrics == nil {
27+
log.L.Warnf("can not collect cache metrics: Metrics is nil")
28+
return
29+
}
30+
31+
prefetchTotalDuration := (c.Metrics.PrefetchEndTimeSecs*1000 +
32+
c.Metrics.PrefetchCumulativeTimeMillis) -
33+
(c.Metrics.PrefetchBeginTimeSecs*1000 +
34+
c.Metrics.PrefetchCumulativeTimeMillis)
35+
36+
data.CachePartialHits.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.PartialHits))
37+
data.CacheWholeHits.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.WholeHits))
38+
data.CacheTotalRequests.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.Total))
39+
data.CacheEntriesCount.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.EntriesCount))
40+
data.CachePrefetchDataBytes.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.PrefetchDataAmount))
41+
data.CachePrefetchRequestsCount.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.PrefetchRequestsCount))
42+
data.CachePrefetchWorkers.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.PrefetchWorkers))
43+
data.CachePrefetchUnmergedChunks.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.PrefetchUnmergedChunks))
44+
data.CachePrefetchCumulativeTimeMillis.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.PrefetchCumulativeTimeMillis))
45+
data.CachePrefetchTotalDurationMillis.WithLabelValues(c.ImageRef).Set(float64(prefetchTotalDuration))
46+
data.CacheBufferedBackendSize.WithLabelValues(c.ImageRef).Set(float64(c.Metrics.BufferedBackendSize))
47+
}
48+
49+
func (c *CacheMetricsVecCollector) Collect() {
50+
for _, cacheMetrics := range c.MetricsVec {
51+
cacheMetrics.Collect()
52+
}
53+
}

‎pkg/metrics/collector/collector.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,11 @@ func NewSnapshotterMetricsCollector(ctx context.Context, cacheDir string, pid in
5656
func NewSnapshotMetricsTimer(method SnapshotMethod) *prometheus.Timer {
5757
return CollectSnapshotMetricsTimer(data.SnapshotEventElapsedHists, method)
5858
}
59+
60+
func NewCacheMetricsCollector(m *types.CacheMetrics, imageRef, daemonID string) *CacheMetricsCollector {
61+
return &CacheMetricsCollector{m, imageRef, daemonID}
62+
}
63+
64+
func NewCacheMetricsVecCollector() *CacheMetricsVecCollector {
65+
return &CacheMetricsVecCollector{}
66+
}

‎pkg/metrics/data/cache.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Copyright (c) 2025. Nydus Developers. All rights reserved.
3+
*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package data
8+
9+
import (
10+
"github.com/containerd/nydus-snapshotter/pkg/metrics/types/ttl"
11+
"github.com/prometheus/client_golang/prometheus"
12+
)
13+
14+
var (
15+
CachePartialHits = ttl.NewGaugeVecWithTTL(
16+
prometheus.GaugeOpts{
17+
Name: "nydusd_cache_partial_hits",
18+
Help: "Number of partial cache hits (IO needs a part of the chunk)",
19+
},
20+
[]string{imageRefLabel},
21+
ttl.DefaultTTL,
22+
)
23+
24+
CacheWholeHits = ttl.NewGaugeVecWithTTL(
25+
prometheus.GaugeOpts{
26+
Name: "nydusd_cache_whole_hits",
27+
Help: "Number of whole cache hits (IO needs the entire chunk)",
28+
},
29+
[]string{imageRefLabel},
30+
ttl.DefaultTTL,
31+
)
32+
33+
CacheTotalRequests = ttl.NewGaugeVecWithTTL(
34+
prometheus.GaugeOpts{
35+
Name: "nydusd_cache_total_requests",
36+
Help: "Total number of cache read requests. Cache hit percentage = (partial_hits + whole_hits) / total",
37+
},
38+
[]string{imageRefLabel},
39+
ttl.DefaultTTL,
40+
)
41+
42+
CacheEntriesCount = ttl.NewGaugeVecWithTTL(
43+
prometheus.GaugeOpts{
44+
Name: "nydusd_cache_entries_count",
45+
Help: "Number of chunks in ready status",
46+
},
47+
[]string{imageRefLabel},
48+
ttl.DefaultTTL,
49+
)
50+
51+
CachePrefetchDataBytes = ttl.NewGaugeVecWithTTL(
52+
prometheus.GaugeOpts{
53+
Name: "nydusd_cache_prefetch_data_bytes",
54+
Help: "Total amount of data prefetched, in bytes",
55+
},
56+
[]string{imageRefLabel},
57+
ttl.DefaultTTL,
58+
)
59+
60+
CachePrefetchRequestsCount = ttl.NewGaugeVecWithTTL(
61+
prometheus.GaugeOpts{
62+
Name: "nydusd_cache_prefetch_requests_count",
63+
Help: "Total prefetch requests issued from storage/blobs or rafs filesystem layer for each file that needs prefetch",
64+
},
65+
[]string{imageRefLabel},
66+
ttl.DefaultTTL,
67+
)
68+
69+
CachePrefetchWorkers = ttl.NewGaugeVecWithTTL(
70+
prometheus.GaugeOpts{
71+
Name: "nydusd_cache_prefetch_workers",
72+
Help: "Number of prefetch workers",
73+
},
74+
[]string{imageRefLabel},
75+
ttl.DefaultTTL,
76+
)
77+
78+
CachePrefetchUnmergedChunks = ttl.NewGaugeVecWithTTL(
79+
prometheus.GaugeOpts{
80+
Name: "nydusd_cache_prefetch_unmerged_chunks",
81+
Help: "Number of unmerged chunks",
82+
},
83+
[]string{imageRefLabel},
84+
ttl.DefaultTTL,
85+
)
86+
87+
CachePrefetchCumulativeTimeMillis = ttl.NewGaugeVecWithTTL(
88+
prometheus.GaugeOpts{
89+
Name: "nydusd_cache_prefetch_cumulative_time_millis",
90+
Help: "Cumulative time latencies in milliseconds of each prefetch request which can be handled in parallel. It starts when the request is born including nydusd processing and schedule and end when the chunk is downloaded and stored. The average prefetch latency can be calculated by `prefetch_cumulative_time_millis / prefetch_requests_count`",
91+
},
92+
[]string{imageRefLabel},
93+
ttl.DefaultTTL,
94+
)
95+
96+
CachePrefetchTotalDurationMillis = ttl.NewGaugeVecWithTTL(
97+
prometheus.GaugeOpts{
98+
Name: "nydusd_cache_prefetch_duration_millis",
99+
Help: "Total wall clock duration of the prefetch, in milliseconds",
100+
},
101+
[]string{imageRefLabel},
102+
ttl.DefaultTTL,
103+
)
104+
105+
CacheBufferedBackendSize = ttl.NewGaugeVecWithTTL(
106+
prometheus.GaugeOpts{
107+
Name: "nydusd_cache_buffered_backend_size",
108+
Help: "Size of the buffered backend, in bytes",
109+
},
110+
[]string{imageRefLabel},
111+
ttl.DefaultTTL,
112+
)
113+
)

‎pkg/metrics/data/daemon.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,6 @@ import (
1111
"github.com/prometheus/client_golang/prometheus"
1212
)
1313

14-
var (
15-
nydusdEventLabel = "nydusd_event"
16-
nydusdVersionLabel = "version"
17-
daemonIDLabel = "daemon_id"
18-
)
19-
2014
var (
2115
NydusdEventCount = prometheus.NewCounterVec(
2216
prometheus.CounterOpts{

‎pkg/metrics/data/fs.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,6 @@ import (
1414
"github.com/prometheus/client_golang/prometheus"
1515
)
1616

17-
var (
18-
imageRefLabel = "image_ref"
19-
)
20-
2117
var (
2218
FsTotalRead = ttl.NewGaugeVecWithTTL(
2319
prometheus.GaugeOpts{

‎pkg/metrics/data/labels.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package data
2+
3+
const (
4+
imageRefLabel = "image_ref"
5+
nydusdEventLabel = "nydusd_event"
6+
nydusdVersionLabel = "version"
7+
daemonIDLabel = "daemon_id"
8+
snapshotEventLabel = "snapshot_operation"
9+
)

‎pkg/metrics/data/snapshotter.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212

1313
var (
1414
defaultDurationBuckets = []float64{.5, 1, 5, 10, 50, 100, 150, 200, 250, 300, 350, 400, 600, 1000}
15-
snapshotEventLabel = "snapshot_operation"
1615
)
1716

1817
var (

‎pkg/metrics/registry/registry.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,17 @@ func init() {
3333
data.Fds,
3434
data.RunTime,
3535
data.Thread,
36+
data.CachePartialHits,
37+
data.CacheWholeHits,
38+
data.CacheTotalRequests,
39+
data.CacheEntriesCount,
40+
data.CachePrefetchDataBytes,
41+
data.CachePrefetchRequestsCount,
42+
data.CachePrefetchWorkers,
43+
data.CachePrefetchUnmergedChunks,
44+
data.CachePrefetchCumulativeTimeMillis,
45+
data.CachePrefetchTotalDurationMillis,
46+
data.CacheBufferedBackendSize,
3647
)
3748

3849
for _, m := range data.MetricHists {

‎pkg/metrics/serve.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ type Server struct {
2929
managers []*manager.Manager
3030
snCollectors []*collector.SnapshotterMetricsCollector
3131
fsCollector *collector.FsMetricsVecCollector
32+
cacheCollector *collector.CacheMetricsVecCollector
3233
inflightCollector *collector.InflightMetricsVecCollector
3334
hungIOInterval time.Duration
3435
collectInterval time.Duration
@@ -71,6 +72,7 @@ func NewServer(ctx context.Context, opts ...ServerOpt) (*Server, error) {
7172

7273
s.fsCollector = collector.NewFsMetricsVecCollector()
7374
s.inflightCollector = collector.NewInflightMetricsVecCollector(s.hungIOInterval)
75+
s.cacheCollector = collector.NewCacheMetricsVecCollector()
7476
for _, pm := range s.managers {
7577
snCollector, err := collector.NewSnapshotterMetricsCollector(ctx, pm.CacheDir(), os.Getpid())
7678
if err != nil {
@@ -145,6 +147,47 @@ func (s *Server) CollectFsMetrics(ctx context.Context) {
145147
}
146148
}
147149

150+
func (s *Server) CollectCacheMetrics(ctx context.Context) {
151+
var cacheMetricsVec []collector.CacheMetricsCollector
152+
153+
for _, pm := range s.managers {
154+
daemons := pm.ListDaemons()
155+
for _, d := range daemons {
156+
// Skip daemons that are not serving
157+
if d.State() != types.DaemonStateRunning {
158+
continue
159+
}
160+
161+
for _, i := range d.RafsCache.List() {
162+
var sid string
163+
164+
if d.IsSharedDaemon() {
165+
sid = i.SnapshotID
166+
} else {
167+
sid = ""
168+
}
169+
170+
cacheMetrics, err := d.GetCacheMetrics(sid)
171+
if err != nil {
172+
log.G(ctx).Errorf("failed to get cache metric: %v", err)
173+
continue
174+
}
175+
176+
cacheMetricsVec = append(cacheMetricsVec, collector.CacheMetricsCollector{
177+
Metrics: cacheMetrics,
178+
ImageRef: i.ImageID,
179+
DaemonID: d.ID(),
180+
})
181+
}
182+
}
183+
}
184+
185+
if cacheMetricsVec != nil {
186+
s.cacheCollector.MetricsVec = cacheMetricsVec
187+
s.cacheCollector.Collect()
188+
}
189+
}
190+
148191
func (s *Server) CollectInflightMetrics(ctx context.Context) {
149192
inflightMetricsVec := make([]*types.InflightMetrics, 0, 16)
150193
for _, pm := range s.managers {
@@ -190,6 +233,7 @@ outer:
190233
select {
191234
case <-timer.C:
192235
s.CollectFsMetrics(ctx)
236+
s.CollectCacheMetrics(ctx)
193237
s.CollectDaemonResourceMetrics(ctx)
194238
// Collect snapshotter metrics.
195239
for _, snCollector := range s.snCollectors {

0 commit comments

Comments
 (0)