Added support for Passivtotal, DNSDB Enterprise as well as DNSDB Community, Added Flags for toggling sources and added Config file support for accessing API Keys and enabling sources

README.md

@@ -9,6 +9,8 @@ If you have Go installed and configured (i.e. with `$GOPATH/bin` in your `$PATH`
 
 ```
 go get -u github.com/tomnomnom/assetfinder
+mkdir ~/.config/assetfinder
+curl https://raw.githubusercontent.com/tomnomnom/assetfinder/master/config.yml > ~/.config/assetfinder/config.yml
 ```
 
 Otherwise [download a release for your platform](https://github.com/tomnomnom/assetfinder/releases).

config.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+    "gopkg.in/yaml.v2"
+    "os"
+)
+
+//Config is config.yml's data structure
+type Config struct {
+     Flags struct{
+         BufferOverrun bool `yaml:"BufferOverrun"`
+         CertSpotter bool `yaml:"CertSpotter"`
+         CrtSh bool `yaml:"CrtSh"`
+         DNSDBCommunity bool `yaml:"DNSDBCommunity"`
+         DNSDB bool `yaml:"DNSDB"`
+         Facebook bool `yaml:"Facebook"`
+         FindSubDomains bool `yaml:"FindSubDomains"`
+         HackerTarget bool `yaml:"HackerTarget"`
+         PassiveTotal bool `yaml:"PassiveTotal"`
+         SubsOnly bool `yaml:"subs-only"`
+         ThreatCrowd bool `yaml:"ThreatCrowd"`
+         Urlscan bool `yaml:"Urlscan"`
+         VirusTotal bool `yaml:"VirusTotal"`
+         Wayback bool `yaml:"Wayback"`
+     } `yaml:"flags"`
+
+    Credentials struct{
+        DNSDB struct{
+            APIKey string `yaml:"api-key"`
+        } `yaml:"dnsdb"`
+
+        Facebook struct{
+            APPID string `yaml:"app-id"`
+            AppSecret string `yaml:"app-secret"`
+        } `yaml:"facebook"`
+
+        FindSubDomains struct{
+            APIToken string `yaml:"api-token"`
+        } `yaml:"spyse"`
+
+        PassiveTotal struct {
+            Username string `yaml:"username"`
+            Secret string `yaml:"secret"`
+        } `yaml:"passivetotal"`
+
+        VirusTotal struct {
+            APIKey string `yaml:"api-key"`
+        } `yaml:"virustotal"`
+    } `yaml:"credentials"`
+}
+
+func (cfg *Config)ymlparser() error{
+    homeDir,err := os.UserHomeDir()
+    if err!= nil{
+        return err
+    }
+
+    f, err := os.Open(homeDir + "/.config/assetfinder/config.yml")
+    if err != nil {
+        return err
+    }
+
+    defer f.Close()
+
+    decoder := yaml.NewDecoder(f)
+    err = decoder.Decode(cfg)
+    if err != nil {
+        return err
+    }
+    return nil
+}

config.yml

@@ -0,0 +1,34 @@
+flags:
+    BufferOverrun: true
+    CertSpotter: true
+    CrtSh: true
+    DNSDBCommunity: false # Per Hour only 4 domains as Community Edition has limit of 25 query per hour and each domain takes 6 request for complete result for community edition See=>dnsdbCommunity.go Line 13
+    DNSDB: true
+    Facebook: true
+    FindSubDomains: true
+    HackerTarget: true
+    PassiveTotal: true
+    subs-only: true
+    ThreatCrowd: true
+    Urlscan: true
+    VirusTotal: true
+    Wayback: false # A little too slow :(
+
+credentials:
+    dnsdb:
+        api-key: ""
+
+    facebook:
+        app-id: ""
+        app-secret: ""
+
+    spyse:
+        api-token: ""
+
+    passivetotal:
+        username: ""
+        secret: ""
+
+    virustotal:
+        api-key: ""
+

dnsdb.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"io/ioutil"
+	"strings"
+)
+
+func getDNSDB (domain string) ([]byte, error){
+    query := fmt.Sprintf("*.%s",domain)
+    url := fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/ANY",query)
+    body,err := reqDNSDB(url)
+    if err != nil{
+        return []byte{},nil
+    }
+    return formatJSON(body), nil
+    }
+
+func clean (domain string)(string){			//Some Domains have wildcards like *.domain.tld / *.*.domain.tld
+	return strings.ReplaceAll(domain, "*.", "")
+}
+
+// Function to send Request to DNSDB
+func reqDNSDB (url string) (string,error) {
+    apikey := cfg.Credentials.DNSDB.APIKey
+    // Failing silently if no API Key
+    if apikey == "" {
+        return "",nil
+    }
+
+    req,err := http.NewRequest("GET", url, nil)
+    if err != nil{
+        return "", err
+    }
+
+    req.Header.Set("X-API-Key",apikey)
+    req.Header.Set("Accept", "*/*")
+
+    resp, err := http.DefaultClient.Do(req)
+    if err != nil{
+        return "", err
+    }
+    defer resp.Body.Close()
+
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil{
+        return "", err
+    }
+    bodySlice :=strings.Split(string(body), "\n")
+    if len(bodySlice)!=2{
+        bodystr := strings.Join(bodySlice[1:len(bodySlice)-2],"")
+        return bodystr, nil
+    }
+    bodystr :=""
+    return bodystr, nil
+}
+
+//Had to do it because couldn't Unmarshal it 
+func formatJSON (bodystr string) []byte{
+    occurence := strings.Count(bodystr,"]}}")
+    bodystr = strings.Replace(bodystr,"]}}","]}},\n",occurence-1)
+    body := []byte(bodystr)
+    body = append([]byte("["),body...)
+    body = append(body, []byte("]")...)
+    return body
+}
+
+func fetchDNSDB (domain string) ([]string, error){
+    body, _ := getDNSDB(domain)
+    wrapper := []struct{
+            Obj struct{
+                Count int `json:"count"`
+                TimeFirst int `json:"time_first"`
+                TimeLast int `json:"time_last"`
+                Rrname string `json:"rrname"`
+                Rrtype string `json:"rrtype"`
+                Bailiwick string `json:"bailiwick"`
+                Rdata []string `json:"rdata"`
+            }`json:"obj"`
+    }{}
+    domainRepeatCheck := make(map[string]bool)
+    var domains []string
+    err := json.Unmarshal(body, &wrapper)
+    if err != nil {
+        return []string{}, err
+    }
+    for _, objelement := range wrapper{
+	    if objelement.Obj.Rrtype == "CNAME" && !domainRepeatCheck[clean(objelement.Obj.Rdata[0])]{
+		    domainRepeatCheck[clean(objelement.Obj.Rdata[0])]= true
+
+		    tempvar := []byte(clean(objelement.Obj.Rdata[0])) // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+		    tempvar = tempvar[:len(tempvar)-2]  // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+		    domains = append(domains,string(tempvar))
+	    }
+            if domainRepeatCheck[clean(objelement.Obj.Rrname)]{
+                continue
+            }
+            domainRepeatCheck[clean(objelement.Obj.Rrname)]= true
+
+            tempvar := []byte(clean(objelement.Obj.Rrname)) // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+            tempvar = tempvar[:len(tempvar)-2]  // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+            domains = append(domains,string(tempvar))
+	}
+    return domains, nil
+}

dnsdbCommunity.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	//"net/http"
+	//"io/ioutil"
+	//"strings"
+)
+
+func getDNSDBCommunity (domain string) ([]byte, error){
+    query := fmt.Sprintf("*.%s",domain)
+    var url []string
+    //Did not use rtype ANY because number of results > number of result limit(256) in Community Edition  
+    url = append(url ,fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/CNAME",query))
+    url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/A",query))
+    url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/AAAA",query))
+    url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/MX",query))
+    url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/HINFO",query))
+    url = append(url, fmt.Sprintf("https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/%s/NS",query))
+
+    body := ""
+    for _, urlElement := range url{
+        tempVar,err := reqDNSDB(urlElement)
+        if err != nil{
+            return []byte{},err
+        }
+        body = body+tempVar
+    }
+
+    return formatJSON(body), nil
+    }
+
+//=================================
+// ====>FUNCTION FROM dnsdb.go<====
+//=================================
+//// Function to send Request to DNSDB
+//func reqDNSDB (url string) (string,error) {
+//    apikey := "" 
+//    if apikey == "" {
+//        return "",nil
+//    }
+//
+//    req,err := http.NewRequest("GET", url, nil)
+//    if err != nil{
+//        return "", err
+//    }
+//
+//    req.Header.Set("X-API-Key",apikey)
+//    req.Header.Set("Accept", "*/*")
+//
+//    resp, err := http.DefaultClient.Do(req)
+//    if err != nil{
+//        return "", err
+//    }
+//    defer resp.Body.Close()
+//
+//    body, err := ioutil.ReadAll(resp.Body)
+//    //body, err := ioutil.ReadFile("data")
+//    if err != nil{
+//        return "", err
+//    }
+//    bodySlice :=strings.Split(string(body), "\n")
+//    if len(bodySlice)!=2{
+//        bodystr := strings.Join(bodySlice[1:len(bodySlice)-2],"")
+//        return bodystr, nil
+//    }
+//    bodystr :=""
+//    return bodystr, nil
+//}
+//
+//
+//=================================
+// ====>FUNCTION FROM dnsdb.go<====
+//=================================
+////Had to do it because couldn't Unmarshal it 
+//func formatJSON (bodystr string) []byte{
+//    occurence := strings.Count(bodystr,"]}}")
+//    bodystr = strings.Replace(bodystr,"]}}","]}},\n",occurence-1)
+//    body := []byte(bodystr)
+//    body = append([]byte("["),body...)
+//    body = append(body, []byte("]")...)
+//    return body
+//}
+
+func fetchDNSDBCommunity (domain string) ([]string, error){
+    body, _ := getDNSDBCommunity(domain)
+    wrapper := []struct{
+            Obj struct{
+                Count int `json:"count"`
+                TimeFirst int `json:"time_first"`
+                TimeLast int `json:"time_last"`
+                Rrname string `json:"rrname"`
+                Rrtype string `json:"rrtype"`
+                Bailiwick string `json:"bailiwick"`
+                Rdata []string `json:"rdata"`
+            }`json:"obj"`
+    }{}
+    domainRepeatCheck := make(map[string]bool)
+    var domains []string
+    err := json.Unmarshal(body, &wrapper)
+    if err != nil {
+        return []string{}, err
+    }
+    for _, objelement := range wrapper{
+	    if objelement.Obj.Rrtype == "CNAME" && !domainRepeatCheck[clean(objelement.Obj.Rdata[0])]{
+		    domainRepeatCheck[clean(objelement.Obj.Rdata[0])]= true
+
+		    tempvar := []byte(clean(objelement.Obj.Rdata[0])) // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+		    tempvar = tempvar[:len(tempvar)-2]  // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+		    domains = append(domains,string(tempvar))
+	    }
+            if domainRepeatCheck[clean(objelement.Obj.Rrname)]{
+                continue
+            }
+            domainRepeatCheck[clean(objelement.Obj.Rrname)]= true
+
+            tempvar := []byte(clean(objelement.Obj.Rrname)) // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+            tempvar = tempvar[:len(tempvar)-2]  // Removing trailing '.' from subdomains Eg: "www.tesla.com."
+            domains = append(domains,string(tempvar))
+        }
+    return domains, nil
+}

facebook.go

@@ -5,13 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"os"
 )
 
 func fetchFacebook(domain string) ([]string, error) {
 
-	appId := os.Getenv("FB_APP_ID")
-	appSecret := os.Getenv("FB_APP_SECRET")
+	appId := cfg.Credentials.Facebook.APPID
+	appSecret := cfg.Credentials.Facebook.AppSecret
 	if appId == "" || appSecret == "" {
 		// fail silently because it's reasonable not to have
 		// the Facebook API creds

findsubdomains.go

@@ -2,11 +2,9 @@ package main
 
 import (
 	"fmt"
-	"os"
 )
 
-var apiToken = os.Getenv("SPYSE_API_TOKEN")
-
+var apiToken = cfg.Credentials.FindSubDomains.APIToken
 func callSubdomainsAggregateEndpoint(domain string) []string {
 	out := make([]string, 0)
 
@@ -101,7 +99,7 @@ func fetchFindSubDomains(domain string) ([]string, error) {
 
 	out := make([]string, 0)
 
-	apiToken := os.Getenv("SPYSE_API_TOKEN")
+	apiToken := cfg.Credentials.FindSubDomains.APIToken 
 	if apiToken == "" {
 		// Must have an API token
 		return []string{}, nil