Merge pull request #1849 from projectdiscovery/dev

httpx - v1.6.7 (UI Dashboard)

Author: ehsandeep

README.md

@@ -134,13 +134,13 @@ MATCHERS:
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
    -ms, -match-string string[]        match response with specified string (-ms admin)
    -mr, -match-regex string[]         match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (leaseweb, stackpath, cloudfront, fastly, google)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (google, cloudfront, fastly)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
 EXTRACTOR:
    -er, -extract-regex string[]   display response content with matched regex
-   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (url,ipv4,mail)
+   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (ipv4,mail,url)
 
 FILTERS:
    -fc, -filter-code string            filter response with specified status code (-fc 403,401)
@@ -151,7 +151,7 @@ FILTERS:
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-ffc 1494302000)
    -fs, -filter-string string[]        filter response with specified string (-fs admin)
    -fe, -filter-regex string[]         filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (leaseweb, stackpath, cloudfront, fastly, google)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (google, cloudfront, fastly)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
    -strip                              strips all tags in response. supported formats: html,xml (default html)
@@ -178,25 +178,25 @@ UPDATE:
    -duc, -disable-update-check  disable automatic httpx update check
 
 OUTPUT:
-   -o, -output string                  file to write output results
-   -oa, -output-all                    filename to write output results in all formats
-   -sr, -store-response                store http response to output directory
-   -srd, -store-response-dir string    store http response to custom directory
-   -ob, -omit-body                     omit response body in output
-   -csv                                store output in csv format
-   -csvo, -csv-output-encoding string  define output encoding
-   -j, -json                           store output in JSONL(ines) format
-   -irh, -include-response-header      include http response (headers) in JSON output (-json only)
-   -irr, -include-response             include http request/response (headers + body) in JSON output (-json only)
-   -irrb, -include-response-base64     include base64 encoded http request/response in JSON output (-json only)
-   -include-chain                      include redirect http chain in JSON output (-json only)
-   -store-chain                        include http redirect chain in responses (-sr only)
-   -svrc, -store-vision-recon-cluster  include visual recon clusters (-ss and -sr only)
-   -pr, -protocol string               protocol to use (unknown, http11)
+   -o, -output string                     file to write output results
+   -oa, -output-all                       filename to write output results in all formats
+   -sr, -store-response                   store http response to output directory
+   -srd, -store-response-dir string       store http response to custom directory
+   -ob, -omit-body                        omit response body in output
+   -csv                                   store output in csv format
+   -csvo, -csv-output-encoding string     define output encoding
+   -j, -json                              store output in JSONL(ines) format
+   -irh, -include-response-header         include http response (headers) in JSON output (-json only)
+   -irr, -include-response                include http request/response (headers + body) in JSON output (-json only)
+   -irrb, -include-response-base64        include base64 encoded http request/response in JSON output (-json only)
+   -include-chain                         include redirect http chain in JSON output (-json only)
+   -store-chain                           include http redirect chain in responses (-sr only)
+   -svrc, -store-vision-recon-cluster     include visual recon clusters (-ss and -sr only)
+   -pr, -protocol string                  protocol to use (unknown, http11)
+   -fepp, -filter-error-page-path string  path to store filtered error pages (default "filtered_error_page.json")
 
 CONFIGURATIONS:
    -config string                   path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
-   -auth                            configure projectdiscovery cloud (pdcp) api key (default true)
    -r, -resolvers string[]          list of custom resolver (file or comma separated)
    -allow string[]                  allowed list of IP/CIDR's to process (file or comma separated)
    -deny string[]                   denied list of IP/CIDR's to process (file or comma separated)
@@ -237,14 +237,21 @@ DEBUG:
 
 OPTIMIZATIONS:
    -nf, -no-fallback                  display both probed protocol (HTTPS and HTTP)
-   -nfs, -no-fallback-scheme          probe with protocol scheme specified in input
+   -nfs, -no-fallback-scheme          probe with protocol scheme specified in input 
    -maxhr, -max-host-error int        max error count per host before skipping remaining path/s (default 30)
    -e, -exclude string[]              exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)
    -retries int                       number of retries
    -timeout int                       timeout in seconds (default 10)
    -delay value                       duration between each http request (eg: 200ms, 1s) (default -1ns)
    -rsts, -response-size-to-save int  max response size to save in bytes (default 2147483647)
    -rstr, -response-size-to-read int  max response size to read in bytes (default 2147483647)
+
+CLOUD:
+   -auth                           configure projectdiscovery cloud (pdcp) api key (default true)
+   -pd, -dashboard                 upload / view output in projectdiscovery cloud (pdcp) UI dashboard
+   -aid, -asset-id string          upload new assets to existing asset id (optional)
+   -aname, -asset-name string      assets group name to set (optional)
+   -pdu, -dashboard-upload string  upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard
 ```
 
 # Running httpx

cmd/httpx/httpx.go

@@ -1,14 +1,19 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"os"
 	"os/signal"
 	"runtime"
 	"runtime/pprof"
 
+	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/httpx/internal/pdcp"
 	"github.com/projectdiscovery/httpx/runner"
-	errorutil "github.com/projectdiscovery/utils/errors"
+	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
+	_ "github.com/projectdiscovery/utils/pprof"
 )
 
 func main() {
@@ -33,6 +38,30 @@ func main() {
 		}()
 	}
 
+	// validation for local results file upload
+	if options.AssetFileUpload != "" {
+		_ = setupOptionalAssetUpload(options)
+		file, err := os.Open(options.AssetFileUpload)
+		if err != nil {
+			gologger.Fatal().Msgf("Could not open file: %s\n", err)
+		}
+		defer file.Close()
+		dec := json.NewDecoder(file)
+		for dec.More() {
+			var r runner.Result
+			err := dec.Decode(&r)
+			if err != nil {
+				gologger.Fatal().Msgf("Could not decode jsonl file: %s\n", err)
+			}
+			options.OnResult(r)
+		}
+		options.OnClose()
+		return
+	}
+
+	// setup optional asset upload
+	_ = setupOptionalAssetUpload(options)
+
 	httpxRunner, err := runner.New(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
@@ -60,8 +89,54 @@ func main() {
 	httpxRunner.Close()
 }
 
-func init() {
-	if os.Getenv("DEBUG") != "" {
-		errorutil.ShowStackTrace = true
+// setupOptionalAssetUpload is used to setup optional asset upload
+// this is optional and only initialized when explicitly enabled
+func setupOptionalAssetUpload(opts *runner.Options) *pdcp.UploadWriter {
+	var mustEnable bool
+	// enable on multiple conditions
+	if opts.AssetUpload || opts.AssetID != "" || opts.AssetName != "" || pdcp.EnableCloudUpload {
+		mustEnable = true
+	}
+	a := aurora.NewAurora(!opts.NoColor)
+	if !mustEnable {
+		if !pdcp.HideAutoSaveMsg {
+			gologger.Print().Msgf("[%s] UI Dashboard is disabled, Use -dashboard option to enable", a.BrightYellow("WRN"))
+		}
+		return nil
+	}
+	if opts.Screenshot {
+		gologger.Fatal().Msgf("Screenshot option is not supported for dashboard upload yet")
+	}
+	gologger.Info().Msgf("To view results in UI dashboard, visit https://cloud.projectdiscovery.io/assets upon completion.")
+	h := &pdcpauth.PDCPCredHandler{}
+	creds, err := h.GetCreds()
+	if err != nil {
+		if err != pdcpauth.ErrNoCreds && !pdcp.HideAutoSaveMsg {
+			gologger.Verbose().Msgf("Could not get credentials for cloud upload: %s\n", err)
+		}
+		pdcpauth.CheckNValidateCredentials("httpx")
+		return nil
+	}
+	writer, err := pdcp.NewUploadWriterCallback(context.Background(), creds)
+	if err != nil {
+		gologger.Error().Msgf("failed to setup UI dashboard: %s", err)
+		return nil
+	}
+	if writer == nil {
+		gologger.Error().Msgf("something went wrong, could not setup UI dashboard")
+	}
+	opts.OnResult = writer.GetWriterCallback()
+	opts.OnClose = func() {
+		writer.Close()
+	}
+	// add additional metadata
+	if opts.AssetID != "" {
+		// silently ignore
+		_ = writer.SetAssetID(opts.AssetID)
+	}
+	if opts.AssetName != "" {
+		// silently ignore
+		writer.SetAssetGroupName(opts.AssetName)
 	}
+	return writer
 }

go.mod

@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
 	github.com/PuerkitoBio/goquery v1.8.1
-	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/akrylysov/pogreb v0.10.2 // indirect
 	github.com/corona10/goimagehash v1.1.0
 	github.com/go-faker/faker/v4 v4.1.1
 	github.com/go-rod/rod v0.114.0
@@ -17,41 +17,41 @@ require (
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
 	github.com/microcosm-cc/bluemonday v1.0.26
-	github.com/miekg/dns v1.1.56 // indirect
+	github.com/miekg/dns v1.1.59 // indirect
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/asnmap v1.1.1
 	github.com/projectdiscovery/cdncheck v1.1.0
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/dsl v0.1.2
-	github.com/projectdiscovery/fastdialer v0.1.5
+	github.com/projectdiscovery/dsl v0.1.7
+	github.com/projectdiscovery/fastdialer v0.2.1
 	github.com/projectdiscovery/fdmax v0.0.4
 	github.com/projectdiscovery/goconfig v0.0.1
-	github.com/projectdiscovery/goflags v0.1.56
-	github.com/projectdiscovery/gologger v1.1.12
-	github.com/projectdiscovery/hmap v0.0.48
+	github.com/projectdiscovery/goflags v0.1.60
+	github.com/projectdiscovery/gologger v1.1.17
+	github.com/projectdiscovery/hmap v0.0.51
 	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/networkpolicy v0.0.9
-	github.com/projectdiscovery/ratelimit v0.0.45
-	github.com/projectdiscovery/rawhttp v0.1.55
-	github.com/projectdiscovery/retryablehttp-go v1.0.65
+	github.com/projectdiscovery/ratelimit v0.0.49
+	github.com/projectdiscovery/rawhttp v0.1.57
+	github.com/projectdiscovery/retryablehttp-go v1.0.71
 	github.com/projectdiscovery/tlsx v1.1.6
-	github.com/projectdiscovery/useragent v0.0.57
-	github.com/projectdiscovery/utils v0.1.5
-	github.com/projectdiscovery/wappalyzergo v0.1.8
+	github.com/projectdiscovery/useragent v0.0.60
+	github.com/projectdiscovery/utils v0.2.3
+	github.com/projectdiscovery/wappalyzergo v0.1.12
 	github.com/rs/xid v1.5.0
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/stretchr/testify v1.9.0
-	github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968
-	go.etcd.io/bbolt v1.3.7 // indirect
+	github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db
+	go.etcd.io/bbolt v1.3.10 // indirect
 	go.uber.org/multierr v1.11.0
-	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/net v0.26.0
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.16.0
 )
 
-require github.com/weppos/publicsuffix-go v0.30.1
+require github.com/weppos/publicsuffix-go v0.30.2
 
 require (
 	aead.dev/minisign v0.2.0 // indirect
@@ -60,7 +60,7 @@ require (
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
-	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
@@ -69,35 +69,34 @@ require (
 	github.com/charmbracelet/glamour v0.6.0 // indirect
 	github.com/cheggaaa/pb/v3 v3.1.4 // indirect
 	github.com/cloudflare/cfssl v1.6.4 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudflare/circl v1.3.8 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/dlclark/regexp2 v1.8.1 // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/gaissmai/bart v0.9.5 // indirect
-	github.com/gaukas/godicttls v0.0.4 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/certificate-transparency-go v1.1.4 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.1 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kataras/jwt v0.1.8 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/kataras/jwt v0.1.10 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kljensen/snowball v0.8.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mholt/archiver/v3 v3.5.1 // indirect
 	github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -107,35 +106,34 @@ require (
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/pierrec/lz4/v4 v4.1.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/gostruct v0.0.2 // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/projectdiscovery/retryabledns v1.0.65 // indirect
-	github.com/quic-go/quic-go v0.42.0 // indirect
-	github.com/refraction-networking/utls v1.5.4 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.68 // indirect
+	github.com/refraction-networking/utls v1.6.7 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
-	github.com/sashabaranov/go-openai v1.14.2 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.7 // indirect
+	github.com/sashabaranov/go-openai v1.15.3 // indirect
+	github.com/shirou/gopsutil/v3 v3.24.2 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
-	github.com/tidwall/btree v1.6.0 // indirect
-	github.com/tidwall/buntdb v1.3.0 // indirect
-	github.com/tidwall/gjson v1.14.4 // indirect
+	github.com/tidwall/btree v1.7.0 // indirect
+	github.com/tidwall/buntdb v1.3.1 // indirect
+	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
-	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/ysmood/fetchup v0.2.3 // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
@@ -149,12 +147,12 @@ require (
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/oauth2 v0.11.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/term v0.21.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect