diff --git a/README.md b/README.md index 7bf4496..ed532b1 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ If you have Go installed and configured (i.e. with `$GOPATH/bin` in your `$PATH` ``` go get -u github.com/tomnomnom/assetfinder +mkdir ~/.config/assetfinder +curl https://raw.githubusercontent.com/tomnomnom/assetfinder/master/config.yml > ~/.config/assetfinder/config.yml ``` Otherwise [download a release for your platform](https://github.com/tomnomnom/assetfinder/releases). diff --git a/config.go b/config.go new file mode 100644 index 0000000..e2b7240 --- /dev/null +++ b/config.go @@ -0,0 +1,71 @@ +package main + +import ( + "gopkg.in/yaml.v2" + "os" +) + +//Config is config.yml's data structure +type Config struct { + Flags struct{ + BufferOverrun bool `yaml:"BufferOverrun"` + CertSpotter bool `yaml:"CertSpotter"` + CrtSh bool `yaml:"CrtSh"` + DNSDBCommunity bool `yaml:"DNSDBCommunity"` + DNSDB bool `yaml:"DNSDB"` + Facebook bool `yaml:"Facebook"` + FindSubDomains bool `yaml:"FindSubDomains"` + HackerTarget bool `yaml:"HackerTarget"` + PassiveTotal bool `yaml:"PassiveTotal"` + SubsOnly bool `yaml:"subs-only"` + ThreatCrowd bool `yaml:"ThreatCrowd"` + Urlscan bool `yaml:"Urlscan"` + VirusTotal bool `yaml:"VirusTotal"` + Wayback bool `yaml:"Wayback"` + } `yaml:"flags"` + + Credentials struct{ + DNSDB struct{ + APIKey string `yaml:"api-key"` + } `yaml:"dnsdb"` + + Facebook struct{ + APPID string `yaml:"app-id"` + AppSecret string `yaml:"app-secret"` + } `yaml:"facebook"` + + FindSubDomains struct{ + APIToken string `yaml:"api-token"` + } `yaml:"spyse"` + + PassiveTotal struct { + Username string `yaml:"username"` + Secret string `yaml:"secret"` + } `yaml:"passivetotal"` + + VirusTotal struct { + APIKey string `yaml:"api-key"` + } `yaml:"virustotal"` + } `yaml:"credentials"` +} + +func (cfg *Config)ymlparser() error{ + homeDir,err := os.UserHomeDir() + if err!= nil{ + return err + } + + f, err := os.Open(homeDir + "/.config/assetfinder/config.yml") + if err != nil { + return err + } + + defer f.Close() + + decoder := yaml.NewDecoder(f) + err = decoder.Decode(cfg) + if err != nil { + return err + } + return nil +} diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..176a03b --- /dev/null +++ b/config.yml @@ -0,0 +1,34 @@ +flags: + BufferOverrun: true + CertSpotter: true + CrtSh: true + DNSDBCommunity: false # Per Hour only 4 domains as Community Edition has limit of 25 query per hour and each domain takes 6 request for complete result for community edition See=>dnsdbCommunity.go Line 13 + DNSDB: true + Facebook: true + FindSubDomains: true + HackerTarget: true + PassiveTotal: true + subs-only: true + ThreatCrowd: true + Urlscan: true + VirusTotal: true + Wayback: false # A little too slow :( + +credentials: + dnsdb: + api-key: "" + + facebook: + app-id: "" + app-secret: "" + + spyse: + api-token: "" + + passivetotal: + username: "" + secret: "" + + virustotal: + api-key: "" + diff --git a/dnsdb.go b/dnsdb.go new file mode 100644 index 0000000..46d240c --- /dev/null +++ b/dnsdb.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "io/ioutil" + "strings" +) + +func getDNSDB (domain string) ([]byte, error){ + query := fmt.Sprintf("*.%s",domain) + url := fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/ANY",query) + body,err := reqDNSDB(url) + if err != nil{ + return []byte{},nil + } + return formatJSON(body), nil + } + +func clean (domain string)(string){ //Some Domains have wildcards like *.domain.tld / *.*.domain.tld + return strings.ReplaceAll(domain, "*.", "") +} + +// Function to send Request to DNSDB +func reqDNSDB (url string) (string,error) { + apikey := cfg.Credentials.DNSDB.APIKey + // Failing silently if no API Key + if apikey == "" { + return "",nil + } + + req,err := http.NewRequest("GET", url, nil) + if err != nil{ + return "", err + } + + req.Header.Set("X-API-Key",apikey) + req.Header.Set("Accept", "*/*") + + resp, err := http.DefaultClient.Do(req) + if err != nil{ + return "", err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil{ + return "", err + } + bodySlice :=strings.Split(string(body), "\n") + if len(bodySlice)!=2{ + bodystr := strings.Join(bodySlice[1:len(bodySlice)-2],"") + return bodystr, nil + } + bodystr :="" + return bodystr, nil +} + +//Had to do it because couldn't Unmarshal it +func formatJSON (bodystr string) []byte{ + occurence := strings.Count(bodystr,"]}}") + bodystr = strings.Replace(bodystr,"]}}","]}},\n",occurence-1) + body := []byte(bodystr) + body = append([]byte("["),body...) + body = append(body, []byte("]")...) + return body +} + +func fetchDNSDB (domain string) ([]string, error){ + body, _ := getDNSDB(domain) + wrapper := []struct{ + Obj struct{ + Count int `json:"count"` + TimeFirst int `json:"time_first"` + TimeLast int `json:"time_last"` + Rrname string `json:"rrname"` + Rrtype string `json:"rrtype"` + Bailiwick string `json:"bailiwick"` + Rdata []string `json:"rdata"` + }`json:"obj"` + }{} + domainRepeatCheck := make(map[string]bool) + var domains []string + err := json.Unmarshal(body, &wrapper) + if err != nil { + return []string{}, err + } + for _, objelement := range wrapper{ + if objelement.Obj.Rrtype == "CNAME" && !domainRepeatCheck[clean(objelement.Obj.Rdata[0])]{ + domainRepeatCheck[clean(objelement.Obj.Rdata[0])]= true + + tempvar := []byte(clean(objelement.Obj.Rdata[0])) // Removing trailing '.' from subdomains Eg: "www.tesla.com." + tempvar = tempvar[:len(tempvar)-2] // Removing trailing '.' from subdomains Eg: "www.tesla.com." + domains = append(domains,string(tempvar)) + } + if domainRepeatCheck[clean(objelement.Obj.Rrname)]{ + continue + } + domainRepeatCheck[clean(objelement.Obj.Rrname)]= true + + tempvar := []byte(clean(objelement.Obj.Rrname)) // Removing trailing '.' from subdomains Eg: "www.tesla.com." + tempvar = tempvar[:len(tempvar)-2] // Removing trailing '.' from subdomains Eg: "www.tesla.com." + domains = append(domains,string(tempvar)) + } + return domains, nil +} diff --git a/dnsdbCommunity.go b/dnsdbCommunity.go new file mode 100644 index 0000000..e914b34 --- /dev/null +++ b/dnsdbCommunity.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "fmt" + //"net/http" + //"io/ioutil" + //"strings" +) + +func getDNSDBCommunity (domain string) ([]byte, error){ + query := fmt.Sprintf("*.%s",domain) + var url []string + //Did not use rtype ANY because number of results > number of result limit(256) in Community Edition + url = append(url ,fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/CNAME",query)) + url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/A",query)) + url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/AAAA",query)) + url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/MX",query)) + url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/HINFO",query)) + url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/NS",query)) + + body := "" + for _, urlElement := range url{ + tempVar,err := reqDNSDB(urlElement) + if err != nil{ + return []byte{},err + } + body = body+tempVar + } + + return formatJSON(body), nil + } + +//================================= +// ====>FUNCTION FROM dnsdb.go<==== +//================================= +//// Function to send Request to DNSDB +//func reqDNSDB (url string) (string,error) { +// apikey := "" +// if apikey == "" { +// return "",nil +// } +// +// req,err := http.NewRequest("GET", url, nil) +// if err != nil{ +// return "", err +// } +// +// req.Header.Set("X-API-Key",apikey) +// req.Header.Set("Accept", "*/*") +// +// resp, err := http.DefaultClient.Do(req) +// if err != nil{ +// return "", err +// } +// defer resp.Body.Close() +// +// body, err := ioutil.ReadAll(resp.Body) +// //body, err := ioutil.ReadFile("data") +// if err != nil{ +// return "", err +// } +// bodySlice :=strings.Split(string(body), "\n") +// if len(bodySlice)!=2{ +// bodystr := strings.Join(bodySlice[1:len(bodySlice)-2],"") +// return bodystr, nil +// } +// bodystr :="" +// return bodystr, nil +//} +// +// +//================================= +// ====>FUNCTION FROM dnsdb.go<==== +//================================= +////Had to do it because couldn't Unmarshal it +//func formatJSON (bodystr string) []byte{ +// occurence := strings.Count(bodystr,"]}}") +// bodystr = strings.Replace(bodystr,"]}}","]}},\n",occurence-1) +// body := []byte(bodystr) +// body = append([]byte("["),body...) +// body = append(body, []byte("]")...) +// return body +//} + +func fetchDNSDBCommunity (domain string) ([]string, error){ + body, _ := getDNSDBCommunity(domain) + wrapper := []struct{ + Obj struct{ + Count int `json:"count"` + TimeFirst int `json:"time_first"` + TimeLast int `json:"time_last"` + Rrname string `json:"rrname"` + Rrtype string `json:"rrtype"` + Bailiwick string `json:"bailiwick"` + Rdata []string `json:"rdata"` + }`json:"obj"` + }{} + domainRepeatCheck := make(map[string]bool) + var domains []string + err := json.Unmarshal(body, &wrapper) + if err != nil { + return []string{}, err + } + for _, objelement := range wrapper{ + if objelement.Obj.Rrtype == "CNAME" && !domainRepeatCheck[clean(objelement.Obj.Rdata[0])]{ + domainRepeatCheck[clean(objelement.Obj.Rdata[0])]= true + + tempvar := []byte(clean(objelement.Obj.Rdata[0])) // Removing trailing '.' from subdomains Eg: "www.tesla.com." + tempvar = tempvar[:len(tempvar)-2] // Removing trailing '.' from subdomains Eg: "www.tesla.com." + domains = append(domains,string(tempvar)) + } + if domainRepeatCheck[clean(objelement.Obj.Rrname)]{ + continue + } + domainRepeatCheck[clean(objelement.Obj.Rrname)]= true + + tempvar := []byte(clean(objelement.Obj.Rrname)) // Removing trailing '.' from subdomains Eg: "www.tesla.com." + tempvar = tempvar[:len(tempvar)-2] // Removing trailing '.' from subdomains Eg: "www.tesla.com." + domains = append(domains,string(tempvar)) + } + return domains, nil +} diff --git a/facebook.go b/facebook.go index de5916e..5eefe18 100644 --- a/facebook.go +++ b/facebook.go @@ -5,13 +5,12 @@ import ( "errors" "fmt" "net/http" - "os" ) func fetchFacebook(domain string) ([]string, error) { - appId := os.Getenv("FB_APP_ID") - appSecret := os.Getenv("FB_APP_SECRET") + appId := cfg.Credentials.Facebook.APPID + appSecret := cfg.Credentials.Facebook.AppSecret if appId == "" || appSecret == "" { // fail silently because it's reasonable not to have // the Facebook API creds diff --git a/findsubdomains.go b/findsubdomains.go index b992b9f..aabbd5b 100644 --- a/findsubdomains.go +++ b/findsubdomains.go @@ -2,11 +2,9 @@ package main import ( "fmt" - "os" ) -var apiToken = os.Getenv("SPYSE_API_TOKEN") - +var apiToken = cfg.Credentials.FindSubDomains.APIToken func callSubdomainsAggregateEndpoint(domain string) []string { out := make([]string, 0) @@ -101,7 +99,7 @@ func fetchFindSubDomains(domain string) ([]string, error) { out := make([]string, 0) - apiToken := os.Getenv("SPYSE_API_TOKEN") + apiToken := cfg.Credentials.FindSubDomains.APIToken if apiToken == "" { // Must have an API token return []string{}, nil diff --git a/main.go b/main.go index 5ce96d0..8641de7 100644 --- a/main.go +++ b/main.go @@ -14,11 +14,32 @@ import ( "time" ) +var cfg Config + func main() { var subsOnly bool - flag.BoolVar(&subsOnly, "subs-only", false, "Only include subdomains of search domain") + override := make(map[string]*bool) + cfg.ymlparser() + + flag.BoolVar(&subsOnly, "subs-only", cfg.Flags.SubsOnly, "Only include subdomains of search domain") + + // Flag for all Modules... Sorry Little messy + override["fetchCertSpotter"] = flag.Bool("cert", cfg.Flags.CertSpotter, " Toggle CertSpotter source") + override["fetchHackerTarget"] = flag.Bool("ht", cfg.Flags.HackerTarget, " Toggle HackerTarget source") + override["fetchThreatCrowd"] = flag.Bool("t", cfg.Flags.ThreatCrowd, " Toggle ThreatCrowd source") + override["fetchCrtSh"] = flag.Bool("crt", cfg.Flags.CrtSh, " Toggle CrtSh source") + override["fetchDNSDB"] = flag.Bool("d", cfg.Flags.DNSDB, " Toggle DNSDB Enterprise source") + override["fetchDNSDBCommunity"] = flag.Bool("dC", cfg.Flags.DNSDBCommunity, " Toggle DNSDB Community source") + override["fetchFacebook"] = flag.Bool("f", cfg.Flags.Facebook, " Toggle Facebook source") + override["fetchPassiveTotal"] = flag.Bool("p", cfg.Flags.PassiveTotal, " Toggle PassiveTotal source") + override["fetchWayback"] = flag.Bool("w", cfg.Flags.Wayback, " Toggle Wayback source") + override["fetchVirusTotal"] = flag.Bool("v", cfg.Flags.VirusTotal, " Toggle VirusTotal source") + override["fetchFindSubDomains"] = flag.Bool("fs", cfg.Flags.FindSubDomains, " Toggle FindSubDomains source") + override["fetchUrlscan"] = flag.Bool("u", cfg.Flags.Urlscan, " Toggle Urlscan source") + override["fetchBufferOverrun"] = flag.Bool("b", cfg.Flags.BufferOverrun, " Toggle BufferOverrun source") flag.Parse() + var domains io.Reader domains = os.Stdin @@ -34,12 +55,16 @@ func main() { fetchCrtSh, fetchFacebook, //fetchWayback, // A little too slow :( + fetchDNSDBCommunity, // Per Hour only 4 domains as Community Edition has limit of 25 query per hour and each domain takes 6 request for complete result for community edition See=>dnsdbCommunity.go Line 13 + fetchPassiveTotal, + //fetchDNSDB, fetchVirusTotal, fetchFindSubDomains, fetchUrlscan, fetchBufferOverrun, } + sources = toggleSources(override, sources)//Toogle Sources according flags out := make(chan string) var wg sync.WaitGroup @@ -71,10 +96,8 @@ func main() { continue } out <- n - } - }() - } - } + }}() + }} // close the output channel when all the workers are done go func() { diff --git a/passivetotal.go b/passivetotal.go new file mode 100644 index 0000000..6aecff0 --- /dev/null +++ b/passivetotal.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" +) + + +func fetchPassiveTotal (domain string) ([]string,error){ + url := fmt.Sprintf("https://api.passivetotal.org/v2/enrichment/subdomains?query=%s",domain) + username := cfg.Credentials.PassiveTotal.Username + secret := cfg.Credentials.PassiveTotal.Secret + if username=="" || secret == ""{ + return []string{}, nil + } + wrapper := struct { + PrimaryDomain string `json:"primaryDomain"` + QueryValue string `json:"queryValue"` + Success bool `json:"success"` + Subdomains []string `json:"subdomains"` + }{} + + //Web Request and Response Block + req, err := http.NewRequest("GET", url, nil) + if err != nil{ + return []string{}, nil + } + + req.SetBasicAuth(username, secret) + + resp, err := http.DefaultClient.Do(req) + if err != nil{ + return []string{}, err + } + + defer resp.Body.Close() + + dec := json.NewDecoder(resp.Body) + err = dec.Decode(&wrapper) + if err != nil{ + return []string{}, err + } + + var domains []string + for _,subdomain := range wrapper.Subdomains{ + domains = append(domains, fmt.Sprintf("%v.%v",subdomain,wrapper.PrimaryDomain)) + } + + return domains,nil +} diff --git a/release b/release new file mode 100755 index 0000000..18f8d4f --- /dev/null +++ b/release @@ -0,0 +1,73 @@ +#!/bin/bash +PROJDIR=$(cd `dirname $0`/.. && pwd) + +VERSION="${1}" +TAG="v${VERSION}" +USER="tomnomnom" +REPO="assetfinder" +BINARY="${REPO}" + +if [[ -z "${VERSION}" ]]; then + echo "Usage: ${0} " + exit 1 +fi + +if [[ -z "${GITHUB_TOKEN}" ]]; then + echo "You forgot to set your GITHUB_TOKEN" + exit 2 +fi + +cd ${PROJDIR} + +# Run the tests +go test +if [ $? -ne 0 ]; then + echo "Tests failed. Aborting." + exit 3 +fi + +# Check if tag exists +git fetch --tags +git tag | grep "^${TAG}$" + +if [ $? -ne 0 ]; then + github-release release \ + --user ${USER} \ + --repo ${REPO} \ + --tag ${TAG} \ + --name "${REPO} ${TAG}" \ + --description "${TAG}" \ + --pre-release +fi + + +for ARCH in "amd64" "386"; do + for OS in "darwin" "linux" "windows" "freebsd"; do + + BINFILE="${BINARY}" + + if [[ "${OS}" == "windows" ]]; then + BINFILE="${BINFILE}.exe" + fi + + rm -f ${BINFILE} + + GOOS=${OS} GOARCH=${ARCH} go build -ldflags "-X main.gronVersion=${VERSION}" github.com/${USER}/${REPO} + + if [[ "${OS}" == "windows" ]]; then + ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.zip" + zip ${ARCHIVE} ${BINFILE} + else + ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.tgz" + tar --create --gzip --file=${ARCHIVE} ${BINFILE} + fi + + echo "Uploading ${ARCHIVE}..." + github-release upload \ + --user ${USER} \ + --repo ${REPO} \ + --tag ${TAG} \ + --name "${ARCHIVE}" \ + --file ${PROJDIR}/${ARCHIVE} + done +done diff --git a/toggle.go b/toggle.go new file mode 100644 index 0000000..f2a9ca4 --- /dev/null +++ b/toggle.go @@ -0,0 +1,27 @@ +package main + +import ( + "reflect" + "runtime" + "strings" +) + +func getFuncName (fn fetchFn) string{ + //get Function name from fetchFn type of format "main.FunctionName" + //removing "main." using strings.Split returning it + return strings.Split(runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name(), ".")[1] +} + +func toggleSources (override map[string]*bool, sources []fetchFn) []fetchFn{ + var sourcelist []fetchFn + + for _,source := range sources{ + + if *override[getFuncName(source)]{ //Checking flags for all functions in sources []fetchFn slice + sourcelist = append(sourcelist,source) + } + + } + return sourcelist +} + diff --git a/virustotal.go b/virustotal.go index f346395..e0b3529 100644 --- a/virustotal.go +++ b/virustotal.go @@ -2,12 +2,11 @@ package main import ( "fmt" - "os" ) func fetchVirusTotal(domain string) ([]string, error) { - apiKey := os.Getenv("VT_API_KEY") + apiKey := cfg.Credentials.VirusTotal.APIKey if apiKey == "" { // swallow not having an API key, just // don't fetch