Merge pull request projectdiscovery#7277 from HarshadaGawas05/feat/honeypot-detection

feat: add honeypot detection to reduce scan noise (projectdiscovery#6403)

Author: Mzack9999

README.md

@@ -338,6 +338,11 @@ UPDATE:
    -ud, -update-template-dir string  custom directory to install / update nuclei-templates
    -duc, -disable-update-check       disable automatic nuclei/templates update check
 
+HONEYPOT:
+   -hpd, -honeypot-detect            detect potential honeypot hosts based on match concentration
+   -hpt, -honeypot-threshold int     number of distinct template IDs required to flag a honeypot host (default 15)
+   -shp, -suppress-honeypot          suppress output for flagged honeypot hosts
+
 STATISTICS:
    -stats                    display statistics about the running scan
    -sj, -stats-json          display statistics in JSONL(ines) format

README_CN.md

@@ -291,6 +291,11 @@ UNCOVER引擎:
     -ud, -update-template-dir string     指定模板目录
     -duc, -disable-update-check          禁用nuclei程序与模板更新
 
+HONEYPOT:
+    -hpd, -honeypot-detect            detect potential honeypot hosts based on match concentration
+    -hpt, -honeypot-threshold int     number of distinct template IDs required to flag a honeypot host (default 15)
+    -shp, -suppress-honeypot          suppress output for flagged honeypot hosts
+
 统计：
     -stats                               显示正在扫描的统计信息
     -sj, -stats-json                     将统计信息以JSONL格式输出到文件

README_ES.md

@@ -296,6 +296,11 @@ UPDATE:
    -ud, -update-template-dir string  directorio personalizado para instalar/actualizar nuclei-templates
    -duc, -disable-update-check       deshabilita la comprobación automática de actualizaciones de nuclei/templates
 
+HONEYPOT:
+   -hpd, -honeypot-detect            detect potential honeypot hosts based on match concentration
+   -hpt, -honeypot-threshold int     number of distinct template IDs required to flag a honeypot host (default 15)
+   -shp, -suppress-honeypot          suppress output for flagged honeypot hosts
+
 STATISTICS:
    -stats                    muestra estadísticas sobre el escaneo en ejecución
    -sj, -stats-json          muestra estadísticas en formato JSONL(ines)

README_ID.md

@@ -262,6 +262,11 @@ UPDATE:
    -ud, -update-template-dir string  custom directory to install / update nuclei-templates
    -duc, -disable-update-check       disable automatic nuclei/templates update check
 
+HONEYPOT:
+   -hpd, -honeypot-detect            detect potential honeypot hosts based on match concentration
+   -hpt, -honeypot-threshold int     number of distinct template IDs required to flag a honeypot host (default 15)
+   -shp, -suppress-honeypot          suppress output for flagged honeypot hosts
+
 STATISTICS:
    -stats                    display statistics about the running scan
    -sj, -stats-json          display statistics in JSONL(ines) format

README_KR.md

@@ -261,6 +261,11 @@ UPDATE:
    -ud, -update-template-dir string  nuclei-templates를 설치/업데이트할 사용자 지정 디렉토리
    -duc, -disable-update-check       자동 nuclei/templates 업데이트 확인 비활성화
 
+HONEYPOT:
+   -hpd, -honeypot-detect            detect potential honeypot hosts based on match concentration
+   -hpt, -honeypot-threshold int     number of distinct template IDs required to flag a honeypot host (default 15)
+   -shp, -suppress-honeypot          suppress output for flagged honeypot hosts
+
 STATISTICS:
    -stats                    실행 중인 스캔에 대한 통계 표시
    -sj, -stats-json          JSONL(ines) 형식으로 통계 표시

README_PT-BR.md

@@ -296,6 +296,11 @@ UPDATE:
    -ud, -update-template-dir string  diretório personalizado para instalar/atualizar os nuclei-templates
    -duc, -disable-update-check       desativa a verificação automática de atualizações do nuclei/templates
 
+HONEYPOT:
+   -hpd, -honeypot-detect            detect potential honeypot hosts based on match concentration
+   -hpt, -honeypot-threshold int     number of distinct template IDs required to flag a honeypot host (default 15)
+   -shp, -suppress-honeypot          suppress output for flagged honeypot hosts
+
 STATISTICS:
    -stats                    exibe estatísticas sobre o scan em execução
    -sj, -stats-json          exibe estatísticas no formato JSONL(ines)

cmd/nuclei/main.go

@@ -487,6 +487,12 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
 	)
 
+	flagSet.CreateGroup("Honeypot", "Honeypot",
+		flagSet.BoolVarP(&options.HoneypotDetection, "honeypot-detect", "hpd", false, "detect potential honeypot hosts based on match concentration"),
+		flagSet.IntVarP(&options.HoneypotThreshold, "honeypot-threshold", "hpt", 15, "number of distinct template IDs required to flag a honeypot host"),
+		flagSet.BoolVarP(&options.SuppressHoneypotResults, "suppress-honeypot", "shp", false, "suppress output for flagged honeypot hosts"),
+	)
+
 	flagSet.CreateGroup("stats", "Statistics",
 		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
 		flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),

internal/runner/runner.go

@@ -54,6 +54,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
@@ -97,6 +98,8 @@ type Runner struct {
 	httpStats          *outputstats.Tracker
 	Logger             *gologger.Logger
 
+	honeypotDetector *honeypotdetector.Detector
+
 	//general purpose temporary directory
 	tmpDir          string
 	parser          parser.Parser
@@ -261,6 +264,12 @@ func New(options *types.Options) (*Runner, error) {
 		}
 	}()
 
+	// Initialize honeypot detector (opt-in) so results can be suppressed.
+	var hpDetector *honeypotdetector.Detector
+	if options.HoneypotDetection {
+		hpDetector = honeypotdetector.New(options.HoneypotThreshold)
+	}
+
 	// create the input provider and load the inputs
 	inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir})
 	if err != nil {
@@ -273,6 +282,10 @@ func New(options *types.Options) (*Runner, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create output file")
 	}
+	if hpDetector != nil {
+		outputWriter.SetHoneypotDetector(hpDetector)
+		runner.honeypotDetector = hpDetector
+	}
 	// setup a proxy writer to automatically upload results to PDCP
 	runner.output = runner.setupPDCPUpload(outputWriter)
 	if options.HTTPStats {
@@ -421,6 +434,10 @@ func (r *Runner) Close() {
 	if r.output != nil {
 		r.output.Close()
 	}
+
+	if r.honeypotDetector != nil {
+		r.Logger.Print().Msgf("%s\n", r.honeypotDetector.Summary())
+	}
 	if r.issuesClient != nil {
 		r.issuesClient.Close()
 	}

pkg/output/output.go

@@ -2,10 +2,12 @@ package output
 
 import (
 	"encoding/base64"
+	stderrors "errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"maps"
+	"net"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -27,6 +29,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/model"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
 	protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
@@ -38,6 +41,10 @@ import (
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
+// ErrHoneypotSuppressed is returned by the output writer when a match result is suppressed
+// due to honeypot detection.
+var ErrHoneypotSuppressed = stderrors.New("honeypot suppressed result")
+
 // Writer is an interface which writes output to somewhere for nuclei events.
 type Writer interface {
 	// Close closes the output writer interface
@@ -65,6 +72,9 @@ type StandardWriter struct {
 	timestamp             bool
 	noMetadata            bool
 	matcherStatus         bool
+	honeypotDetector      *honeypotdetector.Detector
+	suppressHoneypot      bool
+	honeypotThreshold     int
 	mutex                 *sync.Mutex
 	aurora                aurora.Aurora
 	outputFile            io.WriteCloser
@@ -265,21 +275,23 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
 	}
 
 	writer := &StandardWriter{
-		json:             options.JSONL,
-		jsonReqResp:      !options.OmitRawRequests,
-		noMetadata:       options.NoMeta,
-		matcherStatus:    options.MatcherStatus,
-		timestamp:        options.Timestamp,
-		aurora:           auroraColorizer,
-		mutex:            &sync.Mutex{},
-		outputFile:       outputFile,
-		traceFile:        traceOutput,
-		errorFile:        errorOutput,
-		severityColors:   colorizer.New(auroraColorizer),
-		storeResponse:    options.StoreResponse,
-		storeResponseDir: options.StoreResponseDir,
-		omitTemplate:     options.OmitTemplate,
-		KeysToRedact:     options.Redact,
+		json:              options.JSONL,
+		jsonReqResp:       !options.OmitRawRequests,
+		noMetadata:        options.NoMeta,
+		matcherStatus:     options.MatcherStatus,
+		timestamp:         options.Timestamp,
+		suppressHoneypot:  options.SuppressHoneypotResults,
+		honeypotThreshold: options.HoneypotThreshold,
+		aurora:            auroraColorizer,
+		mutex:             &sync.Mutex{},
+		outputFile:        outputFile,
+		traceFile:         traceOutput,
+		errorFile:         errorOutput,
+		severityColors:    colorizer.New(auroraColorizer),
+		storeResponse:     options.StoreResponse,
+		storeResponseDir:  options.StoreResponseDir,
+		omitTemplate:      options.OmitTemplate,
+		KeysToRedact:      options.Redact,
 	}
 
 	if v := os.Getenv("DISABLE_STDOUT"); v == "true" || v == "1" {
@@ -289,6 +301,14 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
 	return writer, nil
 }
 
+// SetHoneypotDetector attaches an initialized honeypot detector to the writer.
+func (w *StandardWriter) SetHoneypotDetector(detector *honeypotdetector.Detector) {
+	w.honeypotDetector = detector
+	if detector != nil {
+		w.honeypotThreshold = detector.Threshold()
+	}
+}
+
 func (w *StandardWriter) ResultCount() int {
 	return int(w.resultCount.Load())
 }
@@ -299,6 +319,29 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 		return nil
 	}
 
+	// Honeypot detection is performed only for successful matches.
+	if event.MatcherStatus && w.honeypotDetector != nil {
+		hostKey := event.URL
+		if hostKey == "" && event.Host != "" {
+			hostKey = event.Host
+			if event.Port != "" {
+				hostKey = net.JoinHostPort(event.Host, event.Port)
+			}
+		}
+
+		if hostKey != "" {
+			justFlagged := w.honeypotDetector.RecordMatch(hostKey, event.TemplateID)
+			if justFlagged {
+				normalized := honeypotdetector.NormalizeHostKey(hostKey)
+				gologger.Warning().Msgf("Potential honeypot detected: %s (matched %d distinct templates)", normalized, w.honeypotThreshold)
+			}
+
+			if w.suppressHoneypot && w.honeypotDetector.IsFlagged(hostKey) {
+				return ErrHoneypotSuppressed
+			}
+		}
+	}
+
 	// Enrich the result event with extra metadata on the template-path and url.
 	if event.TemplatePath != "" {
 		event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier)

pkg/protocols/common/helpers/writer/writer.go

@@ -1,14 +1,16 @@
 package writer
 
 import (
+	stderrors "errors"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 )
 
 // WriteResult is a helper for writing results to the output
-func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
+func WriteResult(data *output.InternalWrappedEvent, out output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
 	// Handle the case where no result found for the template.
 	// In this case, we just show misc information about the failed
 	// match for the template.
@@ -17,18 +19,27 @@ func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progre
 	}
 	var matched bool
 	for _, result := range data.Results {
-		if issuesClient != nil {
+		var suppressed bool
+		if err := out.Write(result); err != nil {
+			if stderrors.Is(err, output.ErrHoneypotSuppressed) {
+				suppressed = true
+			} else {
+				gologger.Warning().Msgf("Could not write output event: %s\n", err)
+			}
+		}
+
+		// Only create issues when the result was not suppressed.
+		if issuesClient != nil && !suppressed {
 			if err := issuesClient.CreateIssue(result); err != nil {
 				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
 			}
 		}
-		if err := output.Write(result); err != nil {
-			gologger.Warning().Msgf("Could not write output event: %s\n", err)
-		}
 		if !matched {
 			matched = true
 		}
-		progress.IncrementMatched()
+		if !suppressed {
+			progress.IncrementMatched()
+		}
 	}
 	return matched
 }