Skip to content

Commit e8ca456

Browse files
committed
feat: Introduce shared object cache to reduce retained memory (#538)
1 parent c686a36 commit e8ca456

24 files changed

+1884
-94
lines changed

config/config.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ type Config struct {
127127
Filters map[string]*FiltersConfig
128128
Proxy ProxyConfig
129129
HTTP HTTPConfig
130+
ObjectCache ObjectCacheConfig
130131

131132
// Optional configuration for metrics integrations. Note that unlike the other fields in Config,
132133
// MetricsConfig is not the name of a configuration file section; the actual sections are the
@@ -301,6 +302,19 @@ type HTTPConfig struct {
301302
EnableCompression bool `conf:"HTTP_ENABLE_COMPRESSION"`
302303
}
303304

305+
// ObjectCacheConfig contains configuration parameters for the shared object cache feature.
306+
//
307+
// This corresponds to the [ObjectCache] section in the configuration file.
308+
//
309+
// Since configuration options can be set either programmatically, or from a file, or from environment
310+
// variables, individual fields are not documented here; instead, see the `README.md` section on
311+
// configuration.
312+
type ObjectCacheConfig struct {
313+
Enabled bool `conf:"OBJECT_CACHE_ENABLED"`
314+
MaxObjects ct.OptIntGreaterThanZero `conf:"OBJECT_CACHE_MAX_OBJECTS"`
315+
TTL ct.OptDuration `conf:"OBJECT_CACHE_TTL"`
316+
}
317+
304318
// MetricsConfig contains configurations for optional metrics integrations.
305319
//
306320
// This corresponds to the [Datadog], [Stackdriver], and [Prometheus] sections in the configuration file.

docs/configuration.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,16 @@ To learn more, read [Metrics integrations](./metrics.md).
299299
|------------------|------------------------|:-------:|:--------|-------------------------------------------------------------------------------------------------------------|
300300
| `enableCompression` | `HTTP_ENABLE_COMPRESSION` | Boolean | `false` | When enabled, the Relay Proxy will compress HTTP responses using gzip compression. This can reduce bandwidth usage but may increase CPU usage. |
301301

302+
### File section: `[ObjectCache]`
303+
304+
The object cache is an in-memory cache that deduplicates feature flags and segments across multiple environments, reducing memory usage when multiple filtered or unfiltered environments share the same underlying data.
305+
306+
| Property in file | Environment var | Type | Default | Description |
307+
|------------------|---------------------------|:--------:|:--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
308+
| `enabled` | `OBJECT_CACHE_ENABLED` | Boolean | `false` | Enables the shared object cache. When enabled, flag and segment data is deduplicated across an environment and its filtered variants. |
309+
| `maxObjects` | `OBJECT_CACHE_MAX_OBJECTS`| Number | `10000` | Maximum number of objects (flags and segments) to cache. When this limit is reached, older objects may be evicted. |
310+
| `ttl` | `OBJECT_CACHE_TTL` | Duration | `5m` | Time-to-live for cached objects. Objects that haven't been accessed within this duration may be evicted during cleanup. This helps prevent memory leaks from stale data. |
311+
302312
### Experimental/testing variables
303313

304314
The current version of the Relay Proxy also supports the following environment variables. These do not have an equivalent in a configuration file; they are not intended for production use; and they are not guaranteed to work in any other Relay Proxy versions.

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ require (
2626
github.com/launchdarkly/eventsource v1.10.0
2727
github.com/launchdarkly/go-configtypes v1.2.0
2828
github.com/launchdarkly/go-jsonstream/v3 v3.1.0
29-
github.com/launchdarkly/go-sdk-common/v3 v3.3.0
29+
github.com/launchdarkly/go-sdk-common/v3 v3.4.0
3030
github.com/launchdarkly/go-sdk-events/v3 v3.5.0
3131
github.com/launchdarkly/go-server-sdk-consul/v3 v3.0.0
3232
github.com/launchdarkly/go-server-sdk-dynamodb/v4 v4.0.0
@@ -36,7 +36,7 @@ require (
3636
github.com/launchdarkly/go-test-helpers/v3 v3.1.0
3737
github.com/launchdarkly/opencensus-go-exporter-stackdriver v0.14.5
3838
github.com/pborman/uuid v1.2.1
39-
github.com/prometheus/client_golang v1.22.0 // indirect; override to address CVE-2022-21698
39+
github.com/prometheus/client_golang v1.22.0 // override to address CVE-2022-21698
4040
github.com/stretchr/testify v1.10.0
4141
go.opencensus.io v0.24.0
4242
golang.org/x/sync v0.15.0

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,8 +333,8 @@ github.com/launchdarkly/go-ntlm-proxy-auth v1.0.2 h1:LnChqC/CulrA+N26DF4rjbDjAAR
333333
github.com/launchdarkly/go-ntlm-proxy-auth v1.0.2/go.mod h1:JClffYrl6+qpGXCmWQQA5UNu7xYxPOXo2HzdyYcUcao=
334334
github.com/launchdarkly/go-ntlmssp v1.0.2 h1:cZU0o9Q9wnlTH8Vw3pex0+/sGyNIsj7T1lCzC2gqAcM=
335335
github.com/launchdarkly/go-ntlmssp v1.0.2/go.mod h1:6ZSwvQs+WBrFEsgKFjrRod8Bj/D4WHHSoo7qJGdgD8g=
336-
github.com/launchdarkly/go-sdk-common/v3 v3.3.0 h1:kkf78wcKX+DOXzNjG29i+py/P+XMIw8/mXS7eEWGQwU=
337-
github.com/launchdarkly/go-sdk-common/v3 v3.3.0/go.mod h1:mXFmDGEh4ydK3QilRhrAyKuf9v44VZQWnINyhqbbOd0=
336+
github.com/launchdarkly/go-sdk-common/v3 v3.4.0 h1:GTRulE0G43xdWY1QdjAXJ7QnZ8PMFU8pOWZICCydEtM=
337+
github.com/launchdarkly/go-sdk-common/v3 v3.4.0/go.mod h1:6MNeeP8b2VtsM6I3TbShCHW/+tYh2c+p5dB+ilS69sg=
338338
github.com/launchdarkly/go-sdk-events/v3 v3.5.0 h1:Yav8Thm70dZbO8U1foYwZPf3w60n/lNBRaYeeNM/qg4=
339339
github.com/launchdarkly/go-sdk-events/v3 v3.5.0/go.mod h1:oepYWQ2RvvjfL2WxkE1uJJIuRsIMOP4WIVgUpXRPcNI=
340340
github.com/launchdarkly/go-semver v1.0.3 h1:agIy/RN3SqeQDIfKkl+oFslEdeIs7pgsJBs3CdCcGQM=

internal/cache/config.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package cache
2+
3+
import (
4+
"github.com/launchdarkly/ld-relay/v8/config"
5+
)
6+
7+
// NewCacheConfigFromRelay converts relay configuration to cache configuration
8+
func NewCacheConfigFromRelay(objectCacheConfig config.ObjectCacheConfig) CacheConfig {
9+
cfg := DefaultCacheConfig()
10+
11+
cfg.Enabled = objectCacheConfig.Enabled
12+
cfg.MaxObjects = objectCacheConfig.MaxObjects.GetOrElse(cfg.MaxObjects) // Use default max objects if not set
13+
cfg.TTL = objectCacheConfig.TTL.GetOrElse(cfg.TTL) // Use default TTL if not set
14+
15+
return cfg
16+
}

internal/cache/config_test.go

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package cache
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
ct "github.com/launchdarkly/go-configtypes"
8+
"github.com/launchdarkly/ld-relay/v8/config"
9+
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
func TestNewCacheConfigFromRelay(t *testing.T) {
14+
tests := []struct {
15+
name string
16+
relayConfig config.ObjectCacheConfig
17+
expectedConfig CacheConfig
18+
}{
19+
{
20+
name: "all values set",
21+
relayConfig: config.ObjectCacheConfig{
22+
Enabled: true,
23+
MaxObjects: mustNewOptInt(5000),
24+
TTL: ct.NewOptDuration(10 * time.Minute),
25+
},
26+
expectedConfig: CacheConfig{
27+
Enabled: true,
28+
MaxObjects: 5000,
29+
TTL: 10 * time.Minute,
30+
},
31+
},
32+
{
33+
name: "disabled cache",
34+
relayConfig: config.ObjectCacheConfig{
35+
Enabled: false,
36+
MaxObjects: mustNewOptInt(1000),
37+
TTL: ct.NewOptDuration(2 * time.Minute),
38+
},
39+
expectedConfig: CacheConfig{
40+
Enabled: false,
41+
MaxObjects: 1000,
42+
TTL: 2 * time.Minute,
43+
},
44+
},
45+
{
46+
name: "default values used when not set",
47+
relayConfig: config.ObjectCacheConfig{
48+
Enabled: true,
49+
// MaxObjects and TTL not set - should use defaults
50+
},
51+
expectedConfig: CacheConfig{
52+
Enabled: true,
53+
MaxObjects: 10000, // default value
54+
TTL: 5 * time.Minute, // default value
55+
},
56+
},
57+
{
58+
name: "empty config uses all defaults",
59+
relayConfig: config.ObjectCacheConfig{
60+
// All fields at zero values
61+
},
62+
expectedConfig: CacheConfig{
63+
Enabled: false, // default value
64+
MaxObjects: 10000, // default value
65+
TTL: 5 * time.Minute, // default value
66+
},
67+
},
68+
}
69+
70+
for _, tt := range tests {
71+
t.Run(tt.name, func(t *testing.T) {
72+
result := NewCacheConfigFromRelay(tt.relayConfig)
73+
assert.Equal(t, tt.expectedConfig, result)
74+
})
75+
}
76+
}
77+
78+
func TestDefaultObjectCacheConfig(t *testing.T) {
79+
config := config.ObjectCacheConfig{}
80+
81+
assert.False(t, config.Enabled) // Should be disabled by default for safety
82+
// MaxObjects and TTL are not set, so they will have zero values
83+
}
84+
85+
func TestCacheConfigValidation(t *testing.T) {
86+
// Test that valid configurations are accepted
87+
validConfig := CacheConfig{
88+
Enabled: true,
89+
MaxObjects: 1000,
90+
TTL: time.Minute,
91+
}
92+
assert.True(t, validConfig.Enabled)
93+
assert.Equal(t, 1000, validConfig.MaxObjects)
94+
assert.Equal(t, time.Minute, validConfig.TTL)
95+
96+
// Test edge cases
97+
edgeConfig := CacheConfig{
98+
Enabled: true,
99+
MaxObjects: 1, // Minimum reasonable value
100+
TTL: time.Millisecond, // Very short TTL
101+
}
102+
assert.True(t, edgeConfig.Enabled)
103+
assert.Equal(t, 1, edgeConfig.MaxObjects)
104+
assert.Equal(t, time.Millisecond, edgeConfig.TTL)
105+
}
106+
107+
func TestOptIntGreaterThanZeroIntegration(t *testing.T) {
108+
// Test that the configtypes integration works correctly
109+
optInt := mustNewOptInt(42)
110+
111+
relayConfig := config.ObjectCacheConfig{
112+
Enabled: true,
113+
MaxObjects: optInt,
114+
}
115+
116+
cacheConfig := NewCacheConfigFromRelay(relayConfig)
117+
assert.Equal(t, 42, cacheConfig.MaxObjects)
118+
}
119+
120+
func TestOptDurationIntegration(t *testing.T) {
121+
// Test that the configtypes integration works correctly
122+
optDuration := ct.NewOptDuration(30 * time.Second)
123+
124+
relayConfig := config.ObjectCacheConfig{
125+
Enabled: true,
126+
TTL: optDuration,
127+
}
128+
129+
cacheConfig := NewCacheConfigFromRelay(relayConfig)
130+
assert.Equal(t, 30*time.Second, cacheConfig.TTL)
131+
}
132+
133+
// Helper function that panics on error for test simplicity
134+
func mustNewOptInt(value int) ct.OptIntGreaterThanZero {
135+
opt, err := ct.NewOptIntGreaterThanZero(value)
136+
if err != nil {
137+
panic(err)
138+
}
139+
return opt
140+
}
141+

internal/cache/metrics.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package cache
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"github.com/prometheus/client_golang/prometheus/promauto"
6+
)
7+
8+
var (
9+
// CacheHitCounter tracks the total number of cache hits
10+
CacheHitCounter = promauto.NewCounter(prometheus.CounterOpts{ //nolint:gochecknoglobals
11+
Name: "ld_relay_object_cache_hits_total",
12+
Help: "Total number of object cache hits",
13+
})
14+
15+
// CacheMissCounter tracks the total number of cache misses
16+
CacheMissCounter = promauto.NewCounter(prometheus.CounterOpts{ //nolint:gochecknoglobals
17+
Name: "ld_relay_object_cache_misses_total",
18+
Help: "Total number of object cache misses",
19+
})
20+
21+
// CacheUpdateCounter tracks the total number of cache updates with newer versions
22+
CacheUpdateCounter = promauto.NewCounter(prometheus.CounterOpts{ //nolint:gochecknoglobals
23+
Name: "ld_relay_object_cache_updates_total",
24+
Help: "Total number of cache updates with newer versions",
25+
})
26+
27+
// CacheEvictionCounter tracks the total number of cache evictions
28+
CacheEvictionCounter = promauto.NewCounter(prometheus.CounterOpts{ //nolint:gochecknoglobals
29+
Name: "ld_relay_object_cache_evictions_total",
30+
Help: "Total number of cache evictions due to TTL or capacity limits",
31+
})
32+
33+
// CacheObjectCountGauge tracks the current number of objects in cache
34+
CacheObjectCountGauge = promauto.NewGauge(prometheus.GaugeOpts{ //nolint:gochecknoglobals
35+
Name: "ld_relay_object_cache_objects_current",
36+
Help: "Current number of objects in cache",
37+
})
38+
39+
// CacheHitRateGauge tracks the cache hit rate as a percentage
40+
CacheHitRateGauge = promauto.NewGauge(prometheus.GaugeOpts{ //nolint:gochecknoglobals
41+
Name: "ld_relay_object_cache_hit_rate",
42+
Help: "Cache hit rate as a percentage (0-100)",
43+
})
44+
)
45+
46+
// UpdatePrometheusMetrics updates Prometheus metrics with current cache statistics
47+
48+
func (c *SharedObjectCache) UpdatePrometheusMetrics() {
49+
stats := c.GetStats()
50+
c.UpdatePrometheusMetricsWithStats(stats)
51+
}
52+
53+
// UpdatePrometheusMetricsWithStats updates Prometheus metrics with provided statistics
54+
// This method avoids potential deadlocks when called from within a locked context
55+
56+
func (c *SharedObjectCache) UpdatePrometheusMetricsWithStats(stats CacheStats) {
57+
// Update gauges
58+
CacheObjectCountGauge.Set(float64(stats.ObjectCount))
59+
CacheHitRateGauge.Set(stats.HitRate() * 100) // Convert to percentage
60+
}
61+
62+
// IncrementHitMetrics increments hit-related metrics
63+
func IncrementHitMetrics() {
64+
CacheHitCounter.Inc()
65+
}
66+
67+
// IncrementMissMetrics increments miss-related metrics
68+
func IncrementMissMetrics() {
69+
CacheMissCounter.Inc()
70+
}
71+
72+
// IncrementUpdateMetrics increments update-related metrics
73+
func IncrementUpdateMetrics() {
74+
CacheUpdateCounter.Inc()
75+
}
76+
77+
// IncrementEvictionMetrics increments eviction-related metrics
78+
func IncrementEvictionMetrics() {
79+
CacheEvictionCounter.Inc()
80+
}

internal/cache/package_info.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
// Package cache contains the implementation of a shared in-memory object cache
2+
// with TTL functionality, used to reduce retained memory usage by sharing
3+
// references to common instances of feature flags or segments.
4+
package cache

0 commit comments

Comments
 (0)