From f9ea876a0ffc7ec2e5223738f6b89f4f8802c7e8 Mon Sep 17 00:00:00 2001 From: brujoand Date: Tue, 21 Apr 2020 08:34:00 +0200 Subject: [PATCH 1/4] Adding support for GHE --- README.md | 1 + config.yaml | 1 + core/config.go | 1 + core/git.go | 16 ++++++++++++++-- core/session.go | 18 ++++++++++++++++++ go.mod | 2 +- 6 files changed, 36 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 262205b..34328c1 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ The `config.yaml` file has 6 elements. A [default is provided](https://github.co github_access_tokens: # provide at least one token - 'token one' - 'token two' +github_enterprise_url: '' # url to your github enterprise (optional) slack_webhook: '' # url to your slack webhook. Found secrets will be sent here blacklisted_extensions: [] # list of extensions to ignore blacklisted_paths: [] # list of paths to ignore diff --git a/config.yaml b/config.yaml index f6ccb32..c15956a 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,6 @@ github_access_tokens: - '' +github_enterprise_url: '' slack_webhook: '' blacklisted_extensions: [".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar.gz", ".ttf", ".lock"] blacklisted_paths: ["node_modules{sep}", "vendor{sep}bundle", "vendor{sep}cache"] # use {sep} for the OS' path seperator (i.e. / or \) diff --git a/core/config.go b/core/config.go index e1bfe8c..2d3c91a 100644 --- a/core/config.go +++ b/core/config.go @@ -12,6 +12,7 @@ import ( type Config struct { GitHubAccessTokens []string `yaml:"github_access_tokens"` + GitHubEnterpriseUrl string `yaml:"github_enterprise_url"` SlackWebhook string `yaml:"slack_webhook,omitempty"` BlacklistedExtensions []string `yaml:"blacklisted_extensions"` BlacklistedPaths []string `yaml:"blacklisted_paths"` diff --git a/core/git.go b/core/git.go index 41fe2b3..f554a73 100644 --- a/core/git.go +++ b/core/git.go @@ -2,19 +2,31 @@ package core import ( "context" + "net/url" "time" "gopkg.in/src-d/go-git.v4" ) -func CloneRepository(session *Session, url string, dir string) (*git.Repository, error) { +func CloneRepository(session *Session, rawUrl string, dir string) (*git.Repository, error) { localCtx, cancel := context.WithTimeout(session.Context, time.Duration(*session.Options.CloneRepositoryTimeout)*time.Second) defer cancel() + if len(session.Config.GitHubEnterpriseUrl) > 0 { + githubUrl, err := url.Parse(rawUrl) + if err != nil { + return nil, err + } + + userInfo := url.User(session.Config.GitHubAccessTokens[0]) + githubUrl.User = userInfo + rawUrl = githubUrl.String() + } + repository, err := git.PlainCloneContext(localCtx, dir, false, &git.CloneOptions{ Depth: 1, RecurseSubmodules: git.NoRecurseSubmodules, - URL: url, + URL: rawUrl, SingleBranch: true, Tags: git.NoTags, }) diff --git a/core/session.go b/core/session.go index 932aeb5..efbe3b6 100644 --- a/core/session.go +++ b/core/session.go @@ -5,8 +5,10 @@ import ( "encoding/csv" "fmt" "math/rand" + "net/url" "os" "runtime" + "strings" "sync" "time" @@ -61,6 +63,22 @@ func (s *Session) InitGitHubClients() { tc := oauth2.NewClient(s.Context, ts) client := github.NewClient(tc) + enterpriseUrl := s.Config.GitHubEnterpriseUrl + + if len(enterpriseUrl) > 0 { + baseEndpoint, err := url.Parse(enterpriseUrl) + + if err != nil { + s.Log.Warn("Failed to parse GitHubEnterpriseUrl %s[..]: %s", enterpriseUrl, err) + return + } + + if !strings.HasSuffix(baseEndpoint.Path, "/api/v3/") { + baseEndpoint.Path += "api/v3/" + } + + client.BaseURL = baseEndpoint + } client.UserAgent = fmt.Sprintf("%s v%s", Name, Version) _, _, err := client.Users.Get(s.Context, "") diff --git a/go.mod b/go.mod index a0de0e8..b8899e6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/eth0izzle/shhgit -go 1.12.9 +go 1.13 require ( github.com/fatih/color v1.7.0 From 764a191775ad19d05e56231787a2c496982aed09 Mon Sep 17 00:00:00 2001 From: brujoand Date: Tue, 21 Apr 2020 09:39:13 +0200 Subject: [PATCH 2/4] Only log rate limit info if we're bein rate limited --- core/github.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/github.go b/core/github.go index a5d5fde..51fb3b4 100644 --- a/core/github.go +++ b/core/github.go @@ -43,7 +43,7 @@ func GetRepositories(session *Session) { GetSession().Log.Important("Error getting GitHub events... trying again", err) } - if opt.Page == 0 { + if opt.Page == 0 && resp.Rate.Limit > 0 { session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit) } @@ -137,10 +137,11 @@ func GetRepository(session *Session, id int64) (*github.Repository, error) { repo, resp, err := client.Repositories.GetByID(session.Context, id) if err != nil { + session.Log.Warn("Got error %s", err) return nil, err } - if resp.Rate.Remaining <= 1 { + if resp.Rate.Remaining <= 1 && resp.Rate.Limit > 0 { session.Log.Warn("Token %s[..] rate limited. Reset at %s", client.Token[:10], resp.Rate.Reset) client.RateLimitedUntil = resp.Rate.Reset.Time } From 1969f505fdf3a6dfaabef91d8a45ac74912a23c5 Mon Sep 17 00:00:00 2001 From: brujoand Date: Wed, 22 Apr 2020 08:16:19 +0200 Subject: [PATCH 3/4] Adding a more specific description in README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 34328c1..09ed99f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ You can use the [precompiled binaries](https://www.github.com/eth0izzle/shhgit/r shhgit needs to access the public GitHub API so you will need to obtain and provide an access token. The API has a hard rate limit of 5,000 requests per hour per account, regardless what token is used. The more account-unique tokens you provide, the faster you can process the events. Follow [this guide](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) to generate a token; it doesn't require any scopes or permissions. And then place it under `github_access_tokens` in `config.yaml`. **Note that it is against the GitHub terms to bypass their rate limits. Use multiple tokens at your own risk**. +For GitHub Enterprise (GHE) a token is also needed, but rate limiting is configurable and your token might need read access to the repositories in questions depending on the configuration of your GHE instance. + Unlike other tools, you don't need to pass any targets with shhgit. Simply run `$ shhgit` to start watching GitHub commits and find secrets or sensitive files matching the included 120 signatures. Alternatively, you can forgo the signatures and use shhgit with a search query, e.g. to find all AWS keys you could use `shhgit --search-query AWS_ACCESS_KEY_ID=AKIA` From 84185fdbeeb9e9da0b06eb5063a704a7465e926d Mon Sep 17 00:00:00 2001 From: Anders Brujordet Date: Wed, 5 Aug 2020 15:57:13 +0200 Subject: [PATCH 4/4] Update git.go Fix consistent naming of the github url to checkout --- core/git.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/git.go b/core/git.go index 02665fe..44726ea 100644 --- a/core/git.go +++ b/core/git.go @@ -41,7 +41,7 @@ func CloneRepository(session *Session, rawUrl string, dir string) (*git.Reposito rawUrl = githubUrl.String() } - session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1)) + session.Log.Debug("[%s] Cloning in to %s", rawUrl, strings.Replace(dir, *session.Options.TempDirectory, "", -1)) repository, err := git.PlainCloneContext(localCtx, dir, false, &git.CloneOptions{ Depth: 1, RecurseSubmodules: git.NoRecurseSubmodules, @@ -51,7 +51,7 @@ func CloneRepository(session *Session, rawUrl string, dir string) (*git.Reposito }) if err != nil { - session.Log.Debug("[%s] Cloning failed: %s", url, err.Error()) + session.Log.Debug("[%s] Cloning failed: %s", rawUrl, err.Error()) return nil, err }