@@ -9,20 +9,37 @@ import (
99 "time"
1010
1111 "github.com/flanksource/artifacts"
12+ dutyCtx "github.com/flanksource/duty/context"
13+ "github.com/flanksource/duty/models"
14+ "github.com/google/uuid"
15+ gocache "github.com/patrickmn/go-cache"
16+ "golang.org/x/time/rate"
17+
1218 "github.com/flanksource/canary-checker/api/context"
1319 "github.com/flanksource/canary-checker/api/external"
1420 v1 "github.com/flanksource/canary-checker/api/v1"
1521 "github.com/flanksource/canary-checker/pkg"
1622 "github.com/flanksource/canary-checker/pkg/utils"
17- "github.com/flanksource/duty/models"
18- "github.com/google/uuid"
19- gocache "github.com/patrickmn/go-cache"
23+ )
24+
25+ const (
26+ defaultSecretLookupFailureThreshold = 50
27+ defaultSecretLookupFailureWindow = 30 * time .Minute
2028)
2129
2230var checksCache = gocache .New (5 * time .Minute , 5 * time .Minute )
2331
32+ var secretLookupRateLimiter = rate .NewLimiter (
33+ rate .Limit (float64 (defaultSecretLookupFailureThreshold )/ defaultSecretLookupFailureWindow .Seconds ()),
34+ defaultSecretLookupFailureThreshold ,
35+ )
36+
2437var DisabledChecks []string
2538
39+ type RunChecksMeta struct {
40+ SecretLookupRateLimitSkipped int
41+ }
42+
2643func getDisabledChecks (ctx * context.Context ) (map [string ]struct {}, error ) {
2744 if val , ok := checksCache .Get ("disabledChecks" ); ok {
2845 return val .(map [string ]struct {}), nil
@@ -150,8 +167,13 @@ func sortChecksByDependency(checks []external.Check) ([]external.Check, error) {
150167 return append (unnamedChecks , sorted ... ), nil
151168}
152169
153- // Exec runs the actions specified and returns the results, without saving artifacts
154- func Exec (ctx * context.Context ) ([]* pkg.CheckResult , error ) {
170+ // RunChecksNoPersistence executes checks and returns transformed results without persistence side effects.
171+ //
172+ // Unlike RunChecks, it does not perform canary deletion checks, dependency ordering,
173+ // artifact saving, or final result post-processing (e.g. resultMode handling).
174+ // It exports only check spec metrics (not per-result emitted metrics) and is intended
175+ // for embedded execution paths (e.g. topology lookups and kubernetesResource sub-checks).
176+ func RunChecksNoPersistence (ctx * context.Context ) ([]* pkg.CheckResult , error ) {
155177 var results []* pkg.CheckResult
156178 disabledChecks , err := getDisabledChecks (ctx )
157179 if err != nil {
@@ -174,9 +196,11 @@ func Exec(ctx *context.Context) ([]*pkg.CheckResult, error) {
174196
175197 result := c .Run (ctx )
176198 transformedResults := TransformResults (ctx , result )
177- results = append (results , transformedResults ... )
178- ExportCheckMetrics (ctx , transformedResults , false )
199+ _ , filteredResults := filterSecretLookupRateLimitedResults (ctx , transformedResults )
200+ results = append (results , filteredResults ... )
201+ ExportCheckMetrics (ctx , filteredResults , false )
179202 }
203+
180204 return results , nil
181205}
182206
@@ -189,23 +213,24 @@ func hasDependencies(checks []external.Check) bool {
189213 return false
190214}
191215
192- func RunChecks (ctx * context.Context ) ([]* pkg.CheckResult , error ) {
216+ func RunChecks (ctx * context.Context ) ([]* pkg.CheckResult , RunChecksMeta , error ) {
193217 var results []* pkg.CheckResult
218+ meta := RunChecksMeta {}
194219 disabledChecks , err := getDisabledChecks (ctx )
195220 if err != nil {
196- return nil , fmt .Errorf ("error getting disabled checks: %v" , err )
221+ return nil , meta , fmt .Errorf ("error getting disabled checks: %v" , err )
197222 }
198223
199224 // Check if canary is not marked deleted in DB
200225 if ctx .DB () != nil && ctx .Canary .GetPersistedID () != "" {
201226 var deletedAt sql.NullTime
202227 err := ctx .DB ().Table ("canaries" ).Select ("deleted_at" ).Where ("id = ? and deleted_at < now()" , ctx .Canary .GetPersistedID ()).Scan (& deletedAt ).Error
203228 if err != nil {
204- return nil , fmt .Errorf ("error getting canary: %v" , err )
229+ return nil , meta , fmt .Errorf ("error getting canary: %v" , err )
205230 }
206231
207232 if deletedAt .Valid {
208- return nil , nil
233+ return nil , meta , nil
209234 }
210235 }
211236
@@ -215,7 +240,7 @@ func RunChecks(ctx *context.Context) ([]*pkg.CheckResult, error) {
215240 if hasDependencies (checks ) {
216241 sortedChecks , err := sortChecksByDependency (checks )
217242 if err != nil {
218- return nil , fmt .Errorf ("failed to sort checks: %v" , err )
243+ return nil , meta , fmt .Errorf ("failed to sort checks: %v" , err )
219244 }
220245
221246 for _ , check := range sortedChecks {
@@ -241,11 +266,13 @@ func RunChecks(ctx *context.Context) ([]*pkg.CheckResult, error) {
241266
242267 result := singleRunner .Check (ctx , check )
243268 transformedResults := TransformResults (ctx , result )
244- results = append (results , transformedResults ... )
245- ExportCheckMetrics (ctx , transformedResults , true )
269+ skippedCount , filteredResults := filterSecretLookupRateLimitedResults (ctx , transformedResults )
270+ meta .SecretLookupRateLimitSkipped += skippedCount
271+ results = append (results , filteredResults ... )
272+ ExportCheckMetrics (ctx , filteredResults , true )
246273
247- if check .GetName () != "" && len (transformedResults ) > 0 && transformedResults [0 ].Pass {
248- ctx .SetOutput (check .GetName (), transformedResults [0 ])
274+ if check .GetName () != "" && len (filteredResults ) > 0 && filteredResults [0 ].Pass {
275+ ctx .SetOutput (check .GetName (), filteredResults [0 ])
249276 }
250277 }
251278 } else {
@@ -259,16 +286,18 @@ func RunChecks(ctx *context.Context) ([]*pkg.CheckResult, error) {
259286
260287 result := c .Run (ctx )
261288 transformedResults := TransformResults (ctx , result )
262- results = append (results , transformedResults ... )
263- ExportCheckMetrics (ctx , transformedResults , true )
289+ skippedCount , filteredResults := filterSecretLookupRateLimitedResults (ctx , transformedResults )
290+ meta .SecretLookupRateLimitSkipped += skippedCount
291+ results = append (results , filteredResults ... )
292+ ExportCheckMetrics (ctx , filteredResults , true )
264293 }
265294 }
266295
267296 if err := saveArtifacts (ctx , results ); err != nil {
268297 ctx .Errorf ("error saving artifacts: %v" , err )
269298 }
270299
271- return ProcessResults (ctx , results ), nil
300+ return ProcessResults (ctx , results ), meta , nil
272301}
273302
274303func saveArtifacts (ctx * context.Context , results pkg.Results ) error {
@@ -356,6 +385,48 @@ func TransformResults(ctx *context.Context, in []*pkg.CheckResult) (out []*pkg.C
356385 return out
357386}
358387
388+ func isSecretLookupRateLimitResult (result * pkg.CheckResult ) bool {
389+ if result == nil {
390+ return false
391+ }
392+ if dutyCtx .IsSecretLookupRateLimited (result .ErrorObject ) {
393+ return true
394+ }
395+ return strings .Contains (strings .ToLower (result .Error ), strings .ToLower (dutyCtx .ErrSecretLookupRateLimited .Error ()))
396+ }
397+
398+ func filterSecretLookupRateLimitedResults (ctx * context.Context , in []* pkg.CheckResult ) (int , []* pkg.CheckResult ) {
399+ if len (in ) == 0 {
400+ return 0 , in
401+ }
402+
403+ skipped := 0
404+ out := make ([]* pkg.CheckResult , 0 , len (in ))
405+
406+ for _ , result := range in {
407+ if ! isSecretLookupRateLimitResult (result ) {
408+ out = append (out , result )
409+ continue
410+ }
411+
412+ if secretLookupRateLimiter .Allow () {
413+ skipped ++
414+ ctx .Warnf ("skipping check result due to secret lookup rate limiting for check=%s (threshold=%d window=%s)" , result .GetName (), defaultSecretLookupFailureThreshold , defaultSecretLookupFailureWindow .Round (time .Second ))
415+ continue
416+ }
417+
418+ ctx .Warnf ("secret lookup rate limit threshold exceeded for check=%s (threshold=%d window=%s), recording as failure" , result .GetName (), defaultSecretLookupFailureThreshold , defaultSecretLookupFailureWindow .Round (time .Second ))
419+ result .Invalid = false
420+ result .Pass = false
421+ if result .Error == "" {
422+ result .Error = "secret lookup repeatedly rate limited"
423+ }
424+ out = append (out , result )
425+ }
426+
427+ return skipped , out
428+ }
429+
359430func ProcessResults (ctx * context.Context , results []* pkg.CheckResult ) []* pkg.CheckResult {
360431 if ctx .Canary .Spec .ResultMode == "" {
361432 return results
0 commit comments