Merge pull request #6697 from projectdiscovery/dwisiswant0/fix/hosterrorscache/dup-log-spam-for-permanent-errs

fix(hosterrorscache): dup log spam for permanent errs

Author: dogancanbakir

pkg/core/executors.go

@@ -112,7 +112,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
 
 					match, err := e.executeTemplateOnInput(ctx, template, t.value)
 					if err != nil {
-						e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), t.value.Input, err)
+						e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", template.ID, t.value.Input, err)
 					}
 					results.CompareAndSwap(false, match)
 				}()
@@ -216,7 +216,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
 
 			match, err := e.executeTemplateOnInput(ctx, template, value)
 			if err != nil {
-				e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), value.Input, err)
+				e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", template.ID, value.Input, err)
 			}
 			results.CompareAndSwap(false, match)
 		}(tpl, target, sg)

pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -27,6 +27,7 @@ type CacheInterface interface {
 	Remove(ctx *contextargs.Context)                                          // remove a host from the cache
 	MarkFailed(protoType string, ctx *contextargs.Context, err error)         // record a failure (and cause) for the host
 	MarkFailedOrRemove(protoType string, ctx *contextargs.Context, err error) // record a failure (and cause) for the host or remove it
+	IsPermanentErr(ctx *contextargs.Context, err error) bool                  // return true if the error is permanent for the host
 }
 
 var (
@@ -137,8 +138,9 @@ func (c *Cache) Check(protoType string, ctx *contextargs.Context) bool {
 	defer cache.mu.Unlock()
 
 	if cache.isPermanentErr {
-		// skipping permanent errors is expected so verbose instead of info
-		gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, cache.cause)
+		cache.Do(func() {
+			gologger.Info().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, cache.cause)
+		})
 		return true
 	}
 
@@ -232,6 +234,28 @@ func (c *Cache) MarkFailedOrRemove(protoType string, ctx *contextargs.Context, e
 	_ = c.failedTargets.Set(cacheKey, cache)
 }
 
+// IsPermanentErr returns true if the error is permanent for the host.
+func (c *Cache) IsPermanentErr(ctx *contextargs.Context, err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) {
+		return true
+	}
+
+	cacheKey := c.GetKeyFromContext(ctx, err)
+	cache, cacheErr := c.failedTargets.GetIFPresent(cacheKey)
+	if cacheErr != nil {
+		return false
+	}
+
+	cache.mu.Lock()
+	defer cache.mu.Unlock()
+
+	return cache.isPermanentErr
+}
+
 // GetKeyFromContext returns the key for the cache from the context
 func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string {
 	// Note:

pkg/protocols/http/request_test.go

@@ -274,6 +274,9 @@ func (f *fakeHostErrorsCache) MarkFailedOrRemove(string, *contextargs.Context, e
 // Check always returns true to simulate an already unresponsive host
 func (f *fakeHostErrorsCache) Check(string, *contextargs.Context) bool { return true }
 
+// IsPermanentErr returns false for tests
+func (f *fakeHostErrorsCache) IsPermanentErr(*contextargs.Context, error) bool { return false }
+
 func TestExecuteParallelHTTP_StopAtFirstMatch(t *testing.T) {
 	options := testutils.DefaultOptions
 	testutils.Init(options)