diff --git a/README.md b/README.md index 3a5f5ef..39f8f66 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ github_access_tokens: # provide at least one token - 'token one' - 'token two' slack_webhook: '' # url to your slack webhook. Found secrets will be sent here +telegram_config: + token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' # Bot token + chat_id: "-1001027884121" # chat id blacklisted_extensions: [] # list of extensions to ignore blacklisted_paths: [] # list of paths to ignore blacklisted_entropy_extensions: [] # additional extensions to ignore for entropy checks diff --git a/core/config.go b/core/config.go index 0d18c68..a0adc68 100755 --- a/core/config.go +++ b/core/config.go @@ -13,12 +13,19 @@ import ( type Config struct { GitHubAccessTokens []string `yaml:"github_access_tokens"` SlackWebhook string `yaml:"slack_webhook,omitempty"` + Telegram TelegramConfig `yaml:"telegram_config,omitempty"` BlacklistedExtensions []string `yaml:"blacklisted_extensions"` BlacklistedPaths []string `yaml:"blacklisted_paths"` BlacklistedEntropyExtensions []string `yaml:"blacklisted_entropy_extensions"` Signatures []ConfigSignature `yaml:"signatures"` } +type TelegramConfig struct { + Token string `yaml:"token,omitempty"` + ChatID string `yaml:"chat_id,omitempty"` + AdminID string `yaml:"admin_id,omitempty"` +} + type ConfigSignature struct { Name string `yaml:"name"` Part string `yaml:"part"` diff --git a/core/github.go b/core/github.go index 7be66bf..8c54120 100755 --- a/core/github.go +++ b/core/github.go @@ -44,7 +44,7 @@ func GetRepositories(session *Session) { } if opt.Page == 0 { - session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit) + //session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit) } newEvents := make([]*github.Event, 0, len(events)) diff --git a/core/log.go b/core/log.go index 9f789a6..22dfcf1 100755 --- a/core/log.go +++ b/core/log.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "log" "net/http" "os" "sync" @@ -31,7 +32,7 @@ var LogColors = map[int]*color.Color{ type Logger struct { sync.Mutex - debug bool + debug bool silent bool } @@ -43,7 +44,7 @@ func (l *Logger) SetSilent(d bool) { l.silent = d } -func (l *Logger) Log(level int, format string, args ...interface{}) { +func (l *Logger) Log(level int, format string, file *MatchFile, args ...interface{}) { l.Lock() defer l.Unlock() @@ -67,31 +68,73 @@ func (l *Logger) Log(level int, format string, args ...interface{}) { http.Post(session.Config.SlackWebhook, "application/json", bytes.NewBuffer(jsonValue)) } + if session.Config.Telegram.Token != "" && session.Config.Telegram.ChatID != "" { + caption := fmt.Sprintf(format+"\n", args...) + rcpt := session.Config.Telegram.ChatID + if level != IMPORTANT && session.Config.Telegram.AdminID != "" { + rcpt = session.Config.Telegram.AdminID + } + + if file != nil { + if len(caption) > 1023 { + caption = caption[0:1023] + } + + values := map[string]string{ + "caption": caption, + "chat_id": rcpt, + "parse_mode": "Markdown", + } + requestURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendDocument", session.Config.Telegram.Token) + request, err := NewfileUploadRequest(requestURL, values, "document", file) + if err != nil { + log.Fatal(err) + } + client := &http.Client{} + + client.Do(request) + } else { + values := map[string]string{ + "text": caption, + "chat_id": rcpt, + "parse_mode": "Markdown", + } + jsonValue, _ := json.Marshal(values) + requestURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", session.Config.Telegram.Token) + http.Post(requestURL, "application/json", bytes.NewBuffer(jsonValue)) + } + + } + if level == FATAL { os.Exit(1) } } func (l *Logger) Fatal(format string, args ...interface{}) { - l.Log(FATAL, format, args...) + l.Log(FATAL, format, nil, args...) } func (l *Logger) Error(format string, args ...interface{}) { - l.Log(ERROR, format, args...) + l.Log(ERROR, format, nil, args...) } func (l *Logger) Warn(format string, args ...interface{}) { - l.Log(WARN, format, args...) + l.Log(WARN, format, nil, args...) } func (l *Logger) Important(format string, args ...interface{}) { - l.Log(IMPORTANT, format, args...) + l.Log(IMPORTANT, format, nil, args...) +} + +func (l *Logger) ImportantFile(format string, file *MatchFile, args ...interface{}) { + l.Log(IMPORTANT, format, file, args...) } func (l *Logger) Info(format string, args ...interface{}) { - l.Log(INFO, format, args...) + l.Log(INFO, format, nil, args...) } func (l *Logger) Debug(format string, args ...interface{}) { - l.Log(DEBUG, format, args...) + l.Log(DEBUG, format, nil, args...) } diff --git a/core/match.go b/core/match.go index 07ab317..f07e37e 100755 --- a/core/match.go +++ b/core/match.go @@ -17,7 +17,15 @@ type MatchFile struct { func NewMatchFile(path string) MatchFile { _, filename := filepath.Split(path) extension := filepath.Ext(path) - contents, _ := ioutil.ReadFile(path) + contents, err := ioutil.ReadFile(path) + if err != nil { + return MatchFile{ + Path: path, + Filename: filename, + Extension: extension, + Contents: []byte{}, + } + } return MatchFile{ Path: path, @@ -59,19 +67,3 @@ func (match MatchFile) CanCheckEntropy() bool { return true } - -func GetMatchingFiles(dir string) []MatchFile { - fileList := make([]MatchFile, 0) - maxFileSize := *session.Options.MaximumFileSize * 1024 - - filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { - if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || IsSkippableFile(path) { - return nil - } - - fileList = append(fileList, NewMatchFile(path)) - return nil - }) - - return fileList -} diff --git a/core/options.go b/core/options.go index 3712f9b..00df9e7 100755 --- a/core/options.go +++ b/core/options.go @@ -20,6 +20,7 @@ type Options struct { TempDirectory *string CsvPath *string SearchQuery *string + NoColor *bool } func ParseOptions() (*Options, error) { @@ -37,6 +38,7 @@ func ParseOptions() (*Options, error) { TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"), CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"), SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"), + NoColor: flag.Bool("no-color", false, "Disable color output"), } flag.Parse() diff --git a/core/util.go b/core/util.go index 52d37d8..65e0333 100755 --- a/core/util.go +++ b/core/util.go @@ -1,9 +1,12 @@ package core import ( + "bytes" "crypto/sha1" "encoding/hex" "math" + "mime/multipart" + "net/http" "os" "path/filepath" "strings" @@ -69,3 +72,29 @@ func GetEntropy(data string) (entropy float64) { return entropy } + +func NewfileUploadRequest(uri string, params map[string]string, paramName string, file *MatchFile) (*http.Request, error) { + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile(paramName, file.Filename) + if err != nil { + return nil, err + } + + part.Write(file.Contents) + + for key, val := range params { + _ = writer.WriteField(key, val) + } + err = writer.Close() + if err != nil { + return nil, err + } + request, err := http.NewRequest("POST", uri, body) + if err != nil { + return nil, err + } + request.Header.Add("Content-Type", writer.FormDataContentType()) + return request, nil + +} diff --git a/go.mod b/go.mod index a0de0e8..25bdc36 100755 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/fatih/color v1.7.0 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.0.0 // indirect + github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.9 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 diff --git a/go.sum b/go.sum index 7709a88..9240dca 100755 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4r github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8= +github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/main.go b/main.go index e2af2c0..583d7cc 100755 --- a/main.go +++ b/main.go @@ -4,11 +4,15 @@ import ( "bufio" "bytes" "os" + "path/filepath" "regexp" "strings" "github.com/eth0izzle/shhgit/core" "github.com/fatih/color" + + //_ "net/http/pprof" + "github.com/iancoleman/strcase" ) var session = core.GetSession() @@ -24,7 +28,7 @@ func ProcessRepositories() { repo, err := core.GetRepository(session, repositoryId) if err != nil { - session.Log.Warn("Failed to retrieve repository %d: %s", repositoryId, err) + //session.Log.Warn("Failed to retrieve repository %d: %s", repositoryId, err) continue } @@ -54,8 +58,8 @@ func ProcessGists() { func processRepositoryOrGist(url string) { var ( - matches []string - matchedAny bool = false + matches []string + //matchedAny bool = false ) dir := core.GetTempDir(core.GetHash(url)) @@ -69,7 +73,16 @@ func processRepositoryOrGist(url string) { session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1)) - for _, file := range core.GetMatchingFiles(dir) { + maxFileSize := *session.Options.MaximumFileSize * 1024 + defer os.RemoveAll(dir) + + filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { + if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || core.IsSkippableFile(path) { + return nil + } + + file := core.NewMatchFile(path) + defer os.Remove(file.Path) relativeFileName := strings.Replace(file.Path, *session.Options.TempDirectory, "", -1) if *session.Options.SearchQuery != "" { @@ -81,24 +94,23 @@ func processRepositoryOrGist(url string) { if matches != nil { count := len(matches) m := strings.Join(matches, ", ") - session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m)) + session.Log.ImportantFile("[%s] %d %s for %s in file %s: %s", &file, url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m)) session.WriteToCsv([]string{url, "Search Query", relativeFileName, m}) } } else { for _, signature := range session.Signatures { if matched, part := signature.Match(file); matched { - matchedAny = true if part == core.PartContents { if matches = signature.GetContentsMatches(file); matches != nil { count := len(matches) m := strings.Join(matches, ", ") - session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), relativeFileName, color.YellowString(m)) + session.Log.ImportantFile("#%s\n\n%s\n%d %s for %s: `%s`", &file, strcase.ToCamel(signature.Name()), url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), color.YellowString(m)) session.WriteToCsv([]string{url, signature.Name(), relativeFileName, m}) } } else { if *session.Options.PathChecks { - session.Log.Important("[%s] Matching file %s for %s", url, color.YellowString(relativeFileName), color.GreenString(signature.Name())) + session.Log.ImportantFile("#%s\n\n%s\n`%s`", &file, strcase.ToCamel(signature.Name()), url, color.GreenString(signature.Name())) session.WriteToCsv([]string{url, signature.Name(), relativeFileName, ""}) } @@ -112,7 +124,7 @@ func processRepositoryOrGist(url string) { entropy := core.GetEntropy(scanner.Text()) if entropy >= *session.Options.EntropyThreshold { - session.Log.Important("[%s] Potential secret in %s = %s", url, color.YellowString(relativeFileName), color.GreenString(scanner.Text())) + session.Log.ImportantFile("#PotentialSecret\n\n%s\n`%s`", &file, url, color.GreenString(scanner.Text())) session.WriteToCsv([]string{url, signature.Name(), relativeFileName, scanner.Text()}) } } @@ -123,14 +135,8 @@ func processRepositoryOrGist(url string) { } } - if !matchedAny { - os.Remove(file.Path) - } - } - - if !matchedAny { - os.RemoveAll(dir) - } + return nil + }) } func main() { @@ -140,6 +146,10 @@ func main() { session.Log.Important("Search Query '%s' given. Only returning matching results.", *session.Options.SearchQuery) } + if *session.Options.NoColor == true { + color.NoColor = true + } + go core.GetRepositories(session) go ProcessRepositories() @@ -149,5 +159,6 @@ func main() { } session.Log.Info("Press Ctrl+C to stop and exit.\n") + //log.Fatal(http.ListenAndServe(":8080", nil)) select {} }