Merge branch 'dev' into pr/2226

Author: Mzack9999

.github/auto_assign.yml

@@ -0,0 +1,9 @@
+addReviewers: true
+reviewers:
+  - dogancanbakir
+  - dwisiswant0
+  - mzack9999
+
+numberOfReviewers: 1
+skipKeywords:
+  - '@dependabot'

.github/workflows/stale.yaml

@@ -1,21 +1,41 @@
-name: 'Close stale issues and PR'
+name: 💤 Stale
+
 on:
   schedule:
-    - cron: '30 1 * * *'
+    - cron: "0 0 * * 0" # Weekly
 
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: write # only for delete-branch option
+      issues: write
+      pull-requests: write
     steps:
       - uses: actions/stale@v9
         with:
-          only-labels: "Status: Abandoned, Type: Question"
-          stale-issue-label: stale
-          stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Remove stale label or comment or this will be closed in 90 days.'
-          close-issue-message: 'This issue was closed because it has been stalled for 90 days with no activity.'
-          days-before-stale: 14
-          days-before-close: 90
-          days-before-pr-stale: -1
-          days-before-pr-close: -1
-          stale-pr-message: ''
-          close-pr-message: ''
+          days-before-stale: 90
+          days-before-close: 7
+          stale-issue-label: "Status: Stale"
+          stale-pr-label: "Status: Stale"
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not
+            had recent activity. It will be closed in 7 days if no further
+            activity occurs. Thank you for your contributions!
+          stale-pr-message: >
+            This pull request has been automatically marked as stale due to
+            inactivity. It will be closed in 7 days if no further activity
+            occurs. Please update if you wish to keep it open.
+          close-issue-message: >
+            This issue has been automatically closed due to inactivity. If you
+            think this is a mistake or would like to continue the discussion,
+            please comment or feel free to reopen it.
+          close-pr-message: >
+            This pull request has been automatically closed due to inactivity.
+            If you think this is a mistake or would like to continue working on
+            it, please comment or feel free to reopen it.
+          close-issue-label: "Status: Abandoned"
+          close-pr-label: "Status: Abandoned"
+          exempt-issue-labels: "Status: Abandoned"
+          exempt-pr-labels: "Status: Abandoned"

README.md

@@ -62,7 +62,7 @@
 
 # Installation Instructions
 
-`httpx` requires **go1.21** to install successfully. Run the following command to get the repo:
+`httpx` requires **go >=1.24.0** to install successfully. Run the following command to get the repo:
 
 ```sh
 go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest
@@ -85,10 +85,6 @@ This will display help for the tool. Here are all the switches it supports.
 
 
 ```console
-Usage:
-  ./httpx [flags]
-
-Flags:
 httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.
 
 Usage:
@@ -115,8 +111,9 @@ PROBES:
    -bp, -body-preview     display first N characters of response body (default 100)
    -server, -web-server   display server name
    -td, -tech-detect      display technology in use based on wappalyzer dataset
+   -cff, -custom-fingerprint-file string  path to a custom fingerprint file for technology detection
    -method                display http request method
-   -websocket             display server using websocket
+   -ws, -websocket        display server using websocket
    -ip                    display host ip
    -cname                 display host cname
    -extract-fqdn, -efqdn  get domain and subdomains from response body and header in jsonl/csv output
@@ -203,6 +200,8 @@ OUTPUT:
    -svrc, -store-vision-recon-cluster     include visual recon clusters (-ss and -sr only)
    -pr, -protocol string                  protocol to use (unknown, http11)
    -fepp, -filter-error-page-path string  path to store filtered error pages (default "filtered_error_page.json")
+   -lof, -list-output-fields              list available output field names for filtering
+   -eof, -exclude-output-fields string[]  exclude specified output fields from results
 
 CONFIGURATIONS:
    -config string                   path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
@@ -211,6 +210,7 @@ CONFIGURATIONS:
    -deny string[]                   denied list of IP/CIDR's to process (file or comma separated)
    -sni, -sni-name string           custom TLS SNI name
    -random-agent                    enable Random User-Agent to use (default true)
+   -auto-referer                    set the Referer header to the current URL (default false)
    -H, -header string[]             custom http headers to send with request
    -http-proxy, -proxy string       http proxy to use (eg http://127.0.0.1:8080)
    -unsafe                          send raw requests skipping golang normalization

cmd/functional-test/main.go

@@ -41,7 +41,9 @@ func runFunctionalTests() error {
 	if err != nil {
 		return errors.Wrap(err, "could not open test cases")
 	}
-	defer file.Close()
+	defer func() {
+		_ = file.Close()
+	}()
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {

cmd/integration-test/http.go

@@ -49,7 +49,7 @@ type standardHttpGet struct {
 func (h *standardHttpGet) Execute() error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "This is a test")
+		_, _ = fmt.Fprintf(w, "This is a test")
 		r.Close = true
 	}))
 	var ts *httptest.Server
@@ -100,7 +100,7 @@ func (h *issue276) Execute() error {
 	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		w.Header().Add("Location", ts.URL+"/redirect")
 		w.WriteHeader(302)
-		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+		_, _ = fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -163,7 +163,7 @@ func (h *issue303) Execute() error {
 		// mimic a misconfigured web server behavior declaring gzip body
 		w.Header().Add("Content-Encoding", "gzip")
 		// but sending it uncompressed
-		fmt.Fprint(w, "<html><body>This is a test</body></html>")
+		_, _ = fmt.Fprint(w, "<html><body>This is a test</body></html>")
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -191,7 +191,7 @@ func (h *issue363) Execute() error {
 	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		w.Header().Add("Location", ts.URL+"/redirect")
 		w.WriteHeader(302)
-		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+		_, _ = fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -214,7 +214,7 @@ func (h *issue400) Execute() error {
 	router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		data, _ := io.ReadAll(r.Body)
-		fmt.Fprintf(w, "data received %s", data)
+		_, _ = fmt.Fprintf(w, "data received %s", data)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -238,7 +238,7 @@ func (h *issue414) Execute() error {
 	router.POST(uripath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		data, _ := io.ReadAll(r.Body)
-		fmt.Fprintf(w, "data received %s", data)
+		_, _ = fmt.Fprintf(w, "data received %s", data)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -265,7 +265,7 @@ func (h *titleUnwantedChars) Execute() error {
 	uriPath := "/index"
 	router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		htmlResponse := "<html><head><title>\v\fProject\n\r Discovery\n - Httpx\t></title></head><body>test data</body></html>"
-		fmt.Fprint(w, htmlResponse)
+		_, _ = fmt.Fprint(w, htmlResponse)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -290,7 +290,7 @@ func (h *issue480) Execute() error {
 	uriPath := "////////////////../../../../../../../../etc/passwd"
 	router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		htmlResponse := "<html><body>ok from uri</body></html>"
-		fmt.Fprint(w, htmlResponse)
+		_, _ = fmt.Fprint(w, htmlResponse)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -314,7 +314,7 @@ func (h *customHeader) Execute() error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -341,7 +341,7 @@ func (h *outputMatchCondition) Execute() error {
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		w.WriteHeader(200)
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -365,7 +365,7 @@ func (h *outputFilterCondition) Execute() error {
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		w.WriteHeader(200)
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -390,7 +390,7 @@ func (h *outputAll) Execute() error {
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		w.WriteHeader(200)
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()

cmd/integration-test/library.go

@@ -22,7 +22,9 @@ func (h *httpxLibrary) Execute() error {
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(testFile)
+	defer func() {
+		_ = os.RemoveAll(testFile)
+	}()
 
 	var got string
 
@@ -64,7 +66,9 @@ func (h *httpxLibraryWithStream) Execute() error {
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(testFile)
+	defer func() {
+		_ = os.RemoveAll(testFile)
+	}()
 
 	var got string
 

common/customports/customport.go

@@ -77,14 +77,14 @@ func (c *CustomPorts) Set(value string) error {
 			}
 			highP, err := strconv.Atoi(potentialRange[1])
 			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("Could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1]))
+				return errors.Wrap(err, fmt.Sprintf("could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1]))
 			}
 			if err := checkPortValue(highP); err != nil {
 				return errors.Wrap(err, fmt.Sprintf("last port of your range(%d)", lowP))
 			}
 
 			if lowP > highP {
-				return fmt.Errorf("First value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP)
+				return fmt.Errorf("first value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP)
 			}
 
 			for i := lowP; i <= highP; i++ {

common/httpx/httpx.go

@@ -155,7 +155,7 @@ func New(options *Options) (*HTTPX, error) {
 
 	if httpx.Options.Protocol == "http11" {
 		// disable http2
-		os.Setenv("GODEBUG", "http2client=0")
+		_ = os.Setenv("GODEBUG", "http2client=0")
 		transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
 	}
 
@@ -359,7 +359,7 @@ func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOpti
 func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
 	method := req.Method
 	headers := req.Header
-	targetURL := req.URL.String()
+	targetURL := req.String()
 	body := req.Body
 	options := rawhttp.DefaultOptions
 	options.Timeout = h.Options.Timeout
@@ -435,6 +435,9 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]st
 		userAgent := useragent.PickRandom()
 		r.Header.Set("User-Agent", userAgent.Raw) //nolint
 	}
+	if h.Options.AutoReferer && r.Header.Get("Referer") == "" {
+		r.Header.Set("Referer", r.String())
+	}
 }
 
 func (httpx *HTTPX) setCustomCookies(req *http.Request) {
@@ -448,7 +451,7 @@ func (httpx *HTTPX) setCustomCookies(req *http.Request) {
 func (httpx *HTTPX) Sanitize(respStr string, trimLine, normalizeSpaces bool) string {
 	respStr = httpx.htmlPolicy.Sanitize(respStr)
 	if trimLine {
-		respStr = strings.Replace(respStr, "\n", "", -1)
+		respStr = strings.ReplaceAll(respStr, "\n", "")
 	}
 	if normalizeSpaces {
 		respStr = httputilz.NormalizeSpaces(respStr)

common/httpx/option.go

@@ -12,6 +12,7 @@ import (
 // Options contains configuration options for the client
 type Options struct {
 	RandomAgent      bool
+	AutoReferer      bool
 	DefaultUserAgent string
 	Proxy            string
 	// Deprecated: use Proxy

common/pagetypeclassifier/pagetypeclassifier.go

@@ -3,7 +3,7 @@ package pagetypeclassifier
 import (
 	_ "embed"
 
-	"github.com/jaytaylor/html2text"
+	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
 	"github.com/projectdiscovery/utils/ml/naive_bayes"
 )
 
@@ -31,7 +31,7 @@ func (n *PageTypeClassifier) Classify(html string) string {
 }
 
 func htmlToText(html string) string {
-	text, err := html2text.FromString(html, html2text.Options{TextOnly: true})
+	text, err := htmltomarkdown.ConvertString(html)
 	if err != nil {
 		panic(err)
 	}