diff --git a/comp/trace/agent/impl/agent.go b/comp/trace/agent/impl/agent.go index 36358df95cf98e..34170f6a0c0f71 100644 --- a/comp/trace/agent/impl/agent.go +++ b/comp/trace/agent/impl/agent.go @@ -137,9 +137,17 @@ func NewAgent(deps dependencies) (traceagent.Component, error) { prepGoRuntime(tracecfg) + tracecfg.SecretsRefreshFn = func() (string, error) { + if deps.Secrets == nil { + log.Error("Secrets component not available, cannot trigger refresh") + return "", errors.New("secrets component not available") + } + return deps.Secrets.Refresh(true) + } + c.Agent = pkgagent.NewAgent( ctx, - c.config.Object(), + tracecfg, c.telemetryCollector, statsdCl, deps.Compressor, diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index cb6466355e00fe..aa214dbe58a434 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -543,6 +543,10 @@ func applyDatadogConfig(c *config.AgentConfig, core corecompcfg.Component) error // Default of 4 was chosen through experimentation, but may not be the optimal value. c.MaxSenderRetries = 4 } + if core.IsSet("secret_refresh_on_api_key_failure_interval") { + // Use the global secret refresh interval for throttling API key refresh at the sender level + c.APIKeyRefreshThrottleInterval = time.Duration(core.GetInt("secret_refresh_on_api_key_failure_interval")) * time.Minute + } if core.IsConfigured("apm_config.sync_flushing") { c.SynchronousFlushing = core.GetBool("apm_config.sync_flushing") } diff --git a/pkg/trace/config/config.go b/pkg/trace/config/config.go index 80dd69cc40d7e0..1e5f2cfe29e386 100644 --- a/pkg/trace/config/config.go +++ b/pkg/trace/config/config.go @@ -415,6 +415,8 @@ type AgentConfig struct { // case, the sender will drop failed payloads when it is unable to enqueue // them for another retry. MaxSenderRetries int + // APIKeyRefreshThrottleInterval is the minimum time between API key refresh attempts. + APIKeyRefreshThrottleInterval time.Duration // HTTP Transport used in writer connections. If nil, default transport values will be used. HTTPTransportFunc func() *http.Transport `json:"-"` // ClientStatsFlushInterval specifies the frequency at which the client stats aggregator will flush its buffer. @@ -557,6 +559,11 @@ type AgentConfig struct { // APMMode specifies whether using "edge" APM mode. May support other modes in the future. If unset, it has no impact. APMMode string + + // SecretsRefreshFn is called when a 403 response is received to trigger + // API key refresh from the secrets backend. It blocks until the refresh + // completes and returns a message and any error encountered. + SecretsRefreshFn func() (string, error) `json:"-"` } // RemoteClient client is used to APM Sampling Updates from a remote source. @@ -612,11 +619,12 @@ func New() *AgentConfig { PipeSecurityDescriptor: "D:AI(A;;GA;;;WD)", GUIPort: "5002", - StatsWriter: new(WriterConfig), - TraceWriter: new(WriterConfig), - ConnectionResetInterval: 0, // disabled - MaxSenderRetries: 4, - ClientStatsFlushInterval: 2 * time.Second, // bucket duration (2s) + StatsWriter: new(WriterConfig), + TraceWriter: new(WriterConfig), + ConnectionResetInterval: 0, // disabled + MaxSenderRetries: 4, + APIKeyRefreshThrottleInterval: 2 * time.Minute, + ClientStatsFlushInterval: 2 * time.Second, // bucket duration (2s) StatsdHost: "localhost", StatsdPort: 8125, diff --git a/pkg/trace/writer/sender.go b/pkg/trace/writer/sender.go index 90f958cb323c32..d4a806c2434c06 100644 --- a/pkg/trace/writer/sender.go +++ b/pkg/trace/writer/sender.go @@ -45,18 +45,24 @@ func newSenders(cfg *config.AgentConfig, r eventRecorder, path string, climit, q log.Criticalf("Invalid host endpoint: %q", endpoint.Host) os.Exit(1) } - senders[i] = newSender(&senderConfig{ + scfg := &senderConfig{ client: cfg.NewHTTPClient(), maxConns: int(maxConns), maxQueued: qsize, maxRetries: cfg.MaxSenderRetries, url: url, - apiKey: endpoint.APIKey, recorder: r, userAgent: fmt.Sprintf("Datadog Trace Agent/%s/%s", cfg.AgentVersion, cfg.GitCommit), isMRF: endpoint.IsMRF, MRFFailoverAPM: cfg.MRFFailoverAPM, - }, statsd) + } + apiKeyManager := &apiKeyManager{ + apiKey: endpoint.APIKey, + refreshFn: cfg.SecretsRefreshFn, + throttleInterval: cfg.APIKeyRefreshThrottleInterval, + } + + senders[i] = newSender(scfg, apiKeyManager, statsd) } return senders } @@ -140,8 +146,6 @@ type senderConfig struct { client *config.ResetClient // url specifies the URL to send requests too. url *url.URL - // apiKey specifies the Datadog API key to use. - apiKey string // maxConns specifies the maximum number of allowed concurrent ougoing // connections. maxConns int @@ -162,10 +166,62 @@ type senderConfig struct { MRFFailoverAPM func() bool } +// apiKeyManager handles API Key access for concurrent use +type apiKeyManager struct { + sync.RWMutex + // apiKey specifies the Datadog API key to use. + apiKey string + + // refreshFn triggers blocking API key refresh from the secrets backend + refreshFn func() (string, error) + + // throttleInterval specifies minimum time between refresh calls + throttleInterval time.Duration + + // lastRefresh tracks when the last refresh occurred + lastRefresh time.Time +} + +func (m *apiKeyManager) Get() string { + m.RLock() + defer m.RUnlock() + return m.apiKey +} + +func (m *apiKeyManager) Update(newKey string) { + m.Lock() + defer m.Unlock() + m.apiKey = newKey +} + +func (m *apiKeyManager) refresh() { + if m.refreshFn == nil || m.throttleInterval == 0 { + return + } + + m.Lock() + if time.Since(m.lastRefresh) < m.throttleInterval { + m.Unlock() + log.Debugf("API Key refresh throttled, last refresh was %v ago", time.Since(m.lastRefresh)) + return + } + + // Update the last refresh time before calling refresh to prevent concurrent calls + m.lastRefresh = time.Now() + m.Unlock() + + if result, err := m.refreshFn(); err != nil { + log.Debugf("API Key refresh failed: %v", err) + } else if result != "" { + log.Infof("API Key refresh completed: %s", result) + } +} + // sender is responsible for sending payloads to a given URL. It uses a size-limited // retry queue with a backoff mechanism in case of retriable errors. type sender struct { - cfg *senderConfig + cfg *senderConfig + apiKeyManager *apiKeyManager queue chan *payload // payload queue inflight *atomic.Int32 // inflight payloads @@ -178,14 +234,15 @@ type sender struct { } // newSender returns a new sender based on the given config cfg. -func newSender(cfg *senderConfig, statsd statsd.ClientInterface) *sender { +func newSender(cfg *senderConfig, apiKeyManager *apiKeyManager, statsd statsd.ClientInterface) *sender { s := sender{ - cfg: cfg, - queue: make(chan *payload, cfg.maxQueued), - inflight: atomic.NewInt32(0), - maxRetries: int32(cfg.maxRetries), - statsd: statsd, - enabled: true, + cfg: cfg, + apiKeyManager: apiKeyManager, + queue: make(chan *payload, cfg.maxQueued), + inflight: atomic.NewInt32(0), + maxRetries: int32(cfg.maxRetries), + statsd: statsd, + enabled: true, } for i := 0; i < cfg.maxConns; i++ { go s.loop() @@ -388,7 +445,7 @@ const ( ) func (s *sender) do(req *http.Request) error { - req.Header.Set(headerAPIKey, s.cfg.apiKey) + req.Header.Set(headerAPIKey, s.apiKeyManager.Get()) req.Header.Set(headerUserAgent, s.cfg.userAgent) resp, err := s.cfg.client.Do(req) if err != nil { @@ -404,6 +461,11 @@ func (s *sender) do(req *http.Request) error { } resp.Body.Close() + if resp.StatusCode == http.StatusForbidden { + log.Debugf("API Key invalid (403), triggering secret refresh") + s.apiKeyManager.refresh() + } + if isRetriable(resp.StatusCode) { return &retriableError{ fmt.Errorf("server responded with %q", resp.Status), @@ -419,7 +481,8 @@ func (s *sender) do(req *http.Request) error { // isRetriable reports whether the give HTTP status code should be retried. func isRetriable(code int) bool { - if code == http.StatusRequestTimeout || code == http.StatusTooManyRequests { + // TODO: Double check what response codes are expected from the backend when API Key is invalid + if code == http.StatusRequestTimeout || code == http.StatusTooManyRequests || code == http.StatusForbidden { return true } // 5xx errors can be retried diff --git a/pkg/trace/writer/sender_test.go b/pkg/trace/writer/sender_test.go index 996eb4681a4893..9ae18c5ff49701 100644 --- a/pkg/trace/writer/sender_test.go +++ b/pkg/trace/writer/sender_test.go @@ -65,6 +65,7 @@ func TestMaxConns(t *testing.T) { func TestIsRetriable(t *testing.T) { for code, want := range map[int]bool{ 400: false, + 403: true, 404: false, 408: true, 409: false, @@ -82,32 +83,16 @@ func TestIsRetriable(t *testing.T) { } func TestSender(t *testing.T) { - statsd := &statsd.NoOpClient{} - const climit = 100 - testSenderConfig := func(serverURL string) *senderConfig { - url, err := url.Parse(serverURL + "/") - if err != nil { - t.Fatal(err) - } - cfg := config.New() - cfg.ConnectionResetInterval = 0 - return &senderConfig{ - client: cfg.NewHTTPClient(), - url: url, - maxConns: climit, - maxQueued: 40, - maxRetries: 4, - apiKey: testAPIKey, - userAgent: "testUserAgent", - } - } - t.Run("accept", func(t *testing.T) { assert := assert.New(t) server := newTestServer() defer server.Close() - s := newSender(testSenderConfig(server.URL), statsd) + s, err := newTestSender(server.URL) + assert.NoError(err) + if err != nil { + t.Fatal(err) + } for i := 0; i < 20; i++ { s.Push(expectResponses(200)) } @@ -124,8 +109,9 @@ func TestSender(t *testing.T) { server := newTestServerWithLatency(50 * time.Millisecond) defer server.Close() - s := newSender(testSenderConfig(server.URL), statsd) - for i := 0; i < climit*2; i++ { + s, err := newTestSender(server.URL) + assert.NoError(err) + for i := 0; i < s.cfg.maxConns*2; i++ { // we have to sleep for a bit to yield to the receiver, otherwise // the channel will get immediately full. time.Sleep(time.Millisecond) @@ -133,9 +119,9 @@ func TestSender(t *testing.T) { } s.Stop() - assert.True(server.Peak() <= climit) - assert.Equal(climit*2, server.Total(), "total") - assert.Equal(climit*2, server.Accepted(), "accepted") + assert.True(server.Peak() <= s.cfg.maxConns) + assert.Equal(s.cfg.maxConns*2, server.Total(), "total") + assert.Equal(s.cfg.maxConns*2, server.Accepted(), "accepted") assert.Equal(0, server.Retried(), "retry") assert.Equal(0, server.Failed(), "failed") }) @@ -145,7 +131,8 @@ func TestSender(t *testing.T) { server := newTestServer() defer server.Close() - s := newSender(testSenderConfig(server.URL), statsd) + s, err := newTestSender(server.URL) + assert.NoError(err) for i := 0; i < 20; i++ { s.Push(expectResponses(404)) } @@ -168,7 +155,8 @@ func TestSender(t *testing.T) { return time.Nanosecond } - s := newSender(testSenderConfig(server.URL), statsd) + s, err := newTestSender(server.URL) + assert.NoError(err) s.Push(expectResponses(503, 408, 200)) s.Stop() @@ -184,10 +172,11 @@ func TestSender(t *testing.T) { defer server.Close() defer useBackoffDuration(time.Millisecond)() - s := newSender(testSenderConfig(server.URL), statsd) + s, err := newTestSender(server.URL) + assert.NoError(err) s.Push(expectResponses(503, 503, 200)) for i := 0; i < 20; i++ { - s.Push(expectResponses(403)) + s.Push(expectResponses(404)) } s.Stop() @@ -208,7 +197,8 @@ func TestSender(t *testing.T) { wg.Done() })) defer server.Close() - s := newSender(testSenderConfig(server.URL), statsd) + s, err := newTestSender(server.URL) + assert.NoError(err) s.Push(expectResponses(http.StatusOK)) s.Stop() wg.Wait() @@ -221,9 +211,10 @@ func TestSender(t *testing.T) { defer useBackoffDuration(0)() var recorder mockRecorder - cfg := testSenderConfig(server.URL) - cfg.recorder = &recorder - s := newSender(cfg, statsd) + + s, err := newTestSender(server.URL) + assert.NoError(err) + s.cfg.recorder = &recorder // push a couple of payloads start := time.Now() @@ -231,7 +222,7 @@ func TestSender(t *testing.T) { s.Push(expectResponses(200)) s.Push(expectResponses(200)) for i := 0; i < 4; i++ { - s.Push(expectResponses(403)) + s.Push(expectResponses(404)) } s.Stop() @@ -247,7 +238,7 @@ func TestSender(t *testing.T) { sent := recorder.data(eventTypeSent) assert.Equal(3, len(sent)) for i := 0; i < 3; i++ { - assert.True(sent[i].bytes > len("|403")) + assert.True(sent[i].bytes > len("|404")) assert.NoError(sent[i].err) assert.Equal(1, sent[i].count) assert.True(time.Since(start)-sent[i].duration < time.Second) @@ -256,8 +247,8 @@ func TestSender(t *testing.T) { failed := recorder.data(eventTypeRejected) assert.Equal(4, len(failed)) for i := 0; i < 4; i++ { - assert.True(failed[i].bytes > len("|403")) - assert.Equal("403 Forbidden", failed[i].err.Error()) + assert.True(failed[i].bytes > len("|404")) + assert.Equal("404 Not Found", failed[i].err.Error()) assert.Equal(1, failed[i].count) assert.True(time.Since(start)-failed[i].duration < time.Second) } @@ -274,10 +265,11 @@ func TestSender(t *testing.T) { defer server.Close() } - senders := []*sender{ - newSender(testSenderConfig(servers[0].URL), statsd), - newSender(testSenderConfig(servers[1].URL), statsd), - newSender(testSenderConfig(servers[2].URL), statsd), + senders := make([]*sender, 3) + for i := range 3 { + s, err := newTestSender(servers[i].URL) + assert.NoError(err) + senders[i] = s } // Enable and failover MRF on s1, enable and not failover on s2, disabled on s3 senders[0].cfg.isMRF = true @@ -314,6 +306,101 @@ func TestSender(t *testing.T) { assert.Equal(0, servers[2].Retried(), "retry") assert.Equal(20, servers[2].Failed(), "failed") }) + + t.Run("403_secrets_refresh_fn", func(t *testing.T) { + assert := assert.New(t) + server := newTestServer() + defer server.Close() + + s, err := newTestSender(server.URL) + assert.NoError(err) + + callbackInvoked := false + s.apiKeyManager.refreshFn = func() (string, error) { + callbackInvoked = true + return "secrets refreshed", nil + } + s.apiKeyManager.throttleInterval = 100 * time.Millisecond + + s.Push(expectResponses(403)) + s.Stop() + + assert.True(callbackInvoked, "secrets refresh callback should have been invoked on 403") + }) + t.Run("403_secrets_refresh_fn_nil", func(t *testing.T) { + assert := assert.New(t) + server := newTestServer() + defer server.Close() + + s, err := newTestSender(server.URL) + assert.NoError(err) + + assert.NotPanics(func() { + s.Push(expectResponses(403)) + s.Stop() + }) + }) + + t.Run("403_retries_with_backoff", func(t *testing.T) { + assert := assert.New(t) + server := newTestServer() + defer server.Close() + defer useBackoffDuration(time.Nanosecond)() + + s, err := newTestSender(server.URL) + assert.NoError(err) + + callbackInvoked := false + s.apiKeyManager.refreshFn = func() (string, error) { + callbackInvoked = true + return "secrets refreshed", nil + } + s.apiKeyManager.throttleInterval = 100 * time.Millisecond + + assert.NoError(err) + s.Push(expectResponses(403, 403, 200)) + s.Stop() + + assert.Equal(3, server.Total(), "should have made 3 requests") + assert.Equal(2, server.Retried(), "should have retried twice") + assert.Equal(1, server.Accepted(), "should have succeeded once") + assert.True(callbackInvoked, "secrets refresh callback should have been invoked on 403") + }) + + t.Run("403_throttles_refresh", func(t *testing.T) { + assert := assert.New(t) + server := newTestServer() + defer server.Close() + defer useBackoffDuration(time.Nanosecond)() + + s, err := newTestSender(server.URL) + assert.NoError(err) + + callCount := 0 + s.apiKeyManager.refreshFn = func() (string, error) { + callCount++ + return "secrets refreshed", nil + } + s.apiKeyManager.throttleInterval = 100 * time.Millisecond + + // First 403 should trigger refresh + s.Push(expectResponses(403)) + s.WaitForInflight() + assert.Equal(1, callCount, "first 403 should trigger refresh") + + s.Push(expectResponses(403)) + s.WaitForInflight() + assert.Equal(1, callCount, "second 403 should be throttled") + + // Wait for throttle interval to expire + time.Sleep(110 * time.Millisecond) + + s.Push(expectResponses(403)) + s.WaitForInflight() + assert.Equal(2, callCount, "third 403 after throttle should trigger refresh") + + s.Stop() + }) } func TestPayload(t *testing.T) { @@ -405,3 +492,25 @@ func (r *mockRecorder) recordEvent(t eventType, data *eventData) { r.rejected = append(r.rejected, data) } } + +func newTestSender(serverURL string) (*sender, error) { + url, err := url.Parse(serverURL + "/") + if err != nil { + return nil, err + } + cfg := config.New() + cfg.ConnectionResetInterval = 0 + scfg := &senderConfig{ + client: cfg.NewHTTPClient(), + url: url, + maxConns: 100, + maxQueued: 40, + maxRetries: 4, + userAgent: "testUserAgent", + } + apiKeyManager := &apiKeyManager{ + apiKey: testAPIKey, + } + statsd := &statsd.NoOpClient{} + return newSender(scfg, apiKeyManager, statsd), nil +} diff --git a/pkg/trace/writer/stats.go b/pkg/trace/writer/stats.go index 696b11ca154cc2..1635c6fc6987f7 100644 --- a/pkg/trace/writer/stats.go +++ b/pkg/trace/writer/stats.go @@ -106,9 +106,9 @@ func NewStatsWriter( // UpdateAPIKey updates the API Key, if needed, on Stats Writer senders. func (w *DatadogStatsWriter) UpdateAPIKey(oldKey, newKey string) { for _, s := range w.senders { - if oldKey == s.cfg.apiKey { + if oldKey == s.apiKeyManager.Get() { + s.apiKeyManager.Update(newKey) log.Debugf("API Key updated for stats endpoint=%s", s.cfg.url) - s.cfg.apiKey = newKey } } } diff --git a/pkg/trace/writer/stats_test.go b/pkg/trace/writer/stats_test.go index 48bd30348eb73d..67bedd9da18f4f 100644 --- a/pkg/trace/writer/stats_test.go +++ b/pkg/trace/writer/stats_test.go @@ -427,15 +427,15 @@ func TestStatsWriterUpdateAPIKey(t *testing.T) { assert.NoError(err) assert.Len(sw.senders, 1) - assert.Equal("123", sw.senders[0].cfg.apiKey) + assert.Equal("123", sw.senders[0].apiKeyManager.Get()) assert.Equal(url, sw.senders[0].cfg.url) sw.UpdateAPIKey("invalid", "foo") - assert.Equal("123", sw.senders[0].cfg.apiKey) + assert.Equal("123", sw.senders[0].apiKeyManager.Get()) assert.Equal(url, sw.senders[0].cfg.url) sw.UpdateAPIKey("123", "foo") - assert.Equal("foo", sw.senders[0].cfg.apiKey) + assert.Equal("foo", sw.senders[0].apiKeyManager.Get()) assert.Equal(url, sw.senders[0].cfg.url) srv.Close() } @@ -610,7 +610,6 @@ func (m *mockContainerTagsBuffer) IsEnabled() bool { } func (m *mockContainerTagsBuffer) AsyncEnrichment(containerID string, cb func([]string, error), _ int64) bool { - returnTags := m.returnTags[containerID] var returnErr error if retErrStr := m.returnErr[containerID]; retErrStr != "" { diff --git a/pkg/trace/writer/trace.go b/pkg/trace/writer/trace.go index 8d167e11971e1d..16e1df51414992 100644 --- a/pkg/trace/writer/trace.go +++ b/pkg/trace/writer/trace.go @@ -100,7 +100,8 @@ func NewTraceWriter( telemetryCollector telemetry.TelemetryCollector, statsd statsd.ClientInterface, timing timing.Reporter, - compressor compression.Component) *TraceWriter { + compressor compression.Component, +) *TraceWriter { tw := &TraceWriter{ prioritySampler: prioritySampler, errorsSampler: errorsSampler, @@ -147,9 +148,9 @@ func NewTraceWriter( // UpdateAPIKey updates the API Key, if needed, on Trace Writer senders. func (w *TraceWriter) UpdateAPIKey(oldKey, newKey string) { for _, s := range w.senders { - if oldKey == s.cfg.apiKey { + if oldKey == s.apiKeyManager.Get() { + s.apiKeyManager.Update(newKey) log.Debugf("API Key updated for traces endpoint=%s", s.cfg.url) - s.cfg.apiKey = newKey } } } diff --git a/pkg/trace/writer/trace_test.go b/pkg/trace/writer/trace_test.go index b19a4148e3415f..71c61a4403a10a 100644 --- a/pkg/trace/writer/trace_test.go +++ b/pkg/trace/writer/trace_test.go @@ -466,15 +466,15 @@ func TestTraceWriterUpdateAPIKey(t *testing.T) { assert.NoError(err) assert.Len(tw.senders, 1) - assert.Equal("123", tw.senders[0].cfg.apiKey) + assert.Equal("123", tw.senders[0].apiKeyManager.Get()) assert.Equal(url, tw.senders[0].cfg.url) tw.UpdateAPIKey("invalid", "foo") - assert.Equal("123", tw.senders[0].cfg.apiKey) + assert.Equal("123", tw.senders[0].apiKeyManager.Get()) assert.Equal(url, tw.senders[0].cfg.url) tw.UpdateAPIKey("123", "foo") - assert.Equal("foo", tw.senders[0].cfg.apiKey) + assert.Equal("foo", tw.senders[0].apiKeyManager.Get()) assert.Equal(url, tw.senders[0].cfg.url) } @@ -563,7 +563,7 @@ func BenchmarkMapDelete(b *testing.B) { } for n := 0; n < b.N; n++ { m["_sampling_priority_v1"] = 1 - //delete(m, "_sampling_priority_v1") + // delete(m, "_sampling_priority_v1") } } @@ -584,7 +584,7 @@ func BenchmarkSpanProto(b *testing.B) { }, } for n := 0; n < b.N; n++ { - //proto.Marshal(&s) + // proto.Marshal(&s) s.MarshalVT() } } diff --git a/pkg/trace/writer/tracev1.go b/pkg/trace/writer/tracev1.go index 88f91c6bea6734..f29c301ac89841 100644 --- a/pkg/trace/writer/tracev1.go +++ b/pkg/trace/writer/tracev1.go @@ -78,7 +78,8 @@ func NewTraceWriterV1( telemetryCollector telemetry.TelemetryCollector, statsd statsd.ClientInterface, timing timing.Reporter, - compressor compression.Component) *TraceWriterV1 { + compressor compression.Component, +) *TraceWriterV1 { tw := &TraceWriterV1{ prioritySampler: prioritySampler, errorsSampler: errorsSampler, @@ -126,9 +127,9 @@ func NewTraceWriterV1( // UpdateAPIKey updates the API Key, if needed, on Trace Writer senders. func (w *TraceWriterV1) UpdateAPIKey(oldKey, newKey string) { for _, s := range w.senders { - if oldKey == s.cfg.apiKey { + if oldKey == s.apiKeyManager.Get() { + s.apiKeyManager.Update(newKey) log.Debugf("API Key updated for traces endpoint=%s", s.cfg.url) - s.cfg.apiKey = newKey } } } diff --git a/pkg/trace/writer/tracev1_test.go b/pkg/trace/writer/tracev1_test.go index 8e58b5797b00fa..54079f12d8e96f 100644 --- a/pkg/trace/writer/tracev1_test.go +++ b/pkg/trace/writer/tracev1_test.go @@ -508,14 +508,14 @@ func TestTraceWriterV1UpdateAPIKey(t *testing.T) { assert.NoError(err) assert.Len(tw.senders, 1) - assert.Equal("123", tw.senders[0].cfg.apiKey) + assert.Equal("123", tw.senders[0].apiKeyManager.Get()) assert.Equal(url, tw.senders[0].cfg.url) tw.UpdateAPIKey("invalid", "foo") - assert.Equal("123", tw.senders[0].cfg.apiKey) + assert.Equal("123", tw.senders[0].apiKeyManager.Get()) assert.Equal(url, tw.senders[0].cfg.url) tw.UpdateAPIKey("123", "foo") - assert.Equal("foo", tw.senders[0].cfg.apiKey) + assert.Equal("foo", tw.senders[0].apiKeyManager.Get()) assert.Equal(url, tw.senders[0].cfg.url) } diff --git a/releasenotes/notes/apm-trigger-refresh-on-403-9650714b18d0ab90.yaml b/releasenotes/notes/apm-trigger-refresh-on-403-9650714b18d0ab90.yaml new file mode 100644 index 00000000000000..ca1752f1a19888 --- /dev/null +++ b/releasenotes/notes/apm-trigger-refresh-on-403-9650714b18d0ab90.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + APM: When a 403 is received from the backend, trigger an API Key refresh, + and retry the payload submission.