Skip to content

Commit bbf6d5e

Browse files
authored
Merge pull request #47 from sacha-c/add-github
feat(#9): add github project scanning support
2 parents b045a01 + 2547e1a commit bbf6d5e

27 files changed

+689
-310
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ require (
2626
github.com/go-git/go-billy/v5 v5.5.0 // indirect
2727
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
2828
github.com/golang/protobuf v1.5.3 // indirect
29+
github.com/google/go-github/v68 v68.0.0 // indirect
2930
github.com/google/go-querystring v1.1.0 // indirect
3031
github.com/gorilla/websocket v1.5.3 // indirect
3132
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
5555
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
5656
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
5757
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
58+
github.com/google/go-github/v68 v68.0.0 h1:ZW57zeNZiXTdQ16qrDiZ0k6XucrxZ2CGmoTvcCyQG6s=
59+
github.com/google/go-github/v68 v68.0.0/go.mod h1:K9HAUBovM2sLwM408A18h+wd9vqdLOEqTUCbnRIcx68=
5860
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
5961
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
6062
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

internal/cli/patrol.go

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@ import (
55
"fmt"
66
"os/exec"
77
"sheriff/internal/config"
8-
"sheriff/internal/git"
98
"sheriff/internal/patrol"
10-
"sheriff/internal/repo"
9+
"sheriff/internal/repository/provider"
1110
"sheriff/internal/scanner"
1211
"sheriff/internal/slack"
1312
"strings"
@@ -34,6 +33,7 @@ const reportToSlackChannel = "report-to-slack-channel"
3433
const reportEnableProjectReportToFlag = "report-enable-project-report-to"
3534
const silentReportFlag = "silent"
3635
const gitlabTokenFlag = "gitlab-token"
36+
const githubTokenFlag = "github-token"
3737
const slackTokenFlag = "slack-token"
3838

3939
var necessaryScanners = []string{scanner.OsvCommandName}
@@ -91,6 +91,13 @@ var PatrolFlags = []cli.Flag{
9191
EnvVars: []string{"GITLAB_TOKEN"},
9292
Category: string(Tokens),
9393
},
94+
&cli.StringFlag{
95+
Name: githubTokenFlag,
96+
Usage: "Token to access the Github API.",
97+
Required: true,
98+
EnvVars: []string{"GITHUB_TOKEN"},
99+
Category: string(Tokens),
100+
},
94101
&cli.StringFlag{
95102
Name: slackTokenFlag,
96103
Usage: "Token to access the Slack API.",
@@ -122,23 +129,23 @@ func PatrolAction(cCtx *cli.Context) error {
122129

123130
// Get tokens
124131
gitlabToken := cCtx.String(gitlabTokenFlag)
132+
githubToken := cCtx.String(githubTokenFlag)
125133
slackToken := cCtx.String(slackTokenFlag)
126134

127135
// Create services
128-
gitlabService, err := repo.NewGitlabService(gitlabToken)
136+
repositoryService, err := provider.NewProvider(gitlabToken, githubToken)
129137
if err != nil {
130-
return errors.Join(errors.New("failed to create GitLab service"), err)
138+
return errors.Join(errors.New("failed to create repository service"), err)
131139
}
132140

133141
slackService, err := slack.New(slackToken, config.Verbose)
134142
if err != nil {
135143
return errors.Join(errors.New("failed to create Slack service"), err)
136144
}
137145

138-
gitService := git.New(gitlabToken)
139146
osvService := scanner.NewOsvScanner()
140147

141-
patrolService := patrol.New(gitlabService, slackService, gitService, osvService)
148+
patrolService := patrol.New(repositoryService, slackService, osvService)
142149

143150
// Check whether the necessary scanners are available
144151
missingScanners := getMissingScanners(necessaryScanners)

internal/config/patrol.go

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ import (
44
"errors"
55
"fmt"
66
"net/url"
7-
"sheriff/internal/repo"
7+
"sheriff/internal/repository"
88

99
zerolog "github.com/rs/zerolog/log"
1010
)
1111

1212
type ProjectLocation struct {
13-
Type repo.PlatformType
13+
Type repository.RepositoryType
1414
Path string
1515
}
1616

@@ -124,9 +124,7 @@ func parseTargets(targets []string) ([]ProjectLocation, error) {
124124
return nil, fmt.Errorf("target missing platform scheme %v", t)
125125
}
126126

127-
if parsed.Scheme == string(repo.Github) {
128-
return nil, fmt.Errorf("github is currently unsupported, but is on our roadmap 😃") // TODO #9
129-
} else if parsed.Scheme != string(repo.Gitlab) {
127+
if parsed.Scheme != string(repository.Gitlab) && parsed.Scheme != string(repository.Github) {
130128
return nil, fmt.Errorf("unsupported platform %v", parsed.Scheme)
131129
}
132130

@@ -136,7 +134,7 @@ func parseTargets(targets []string) ([]ProjectLocation, error) {
136134
}
137135

138136
locations[i] = ProjectLocation{
139-
Type: repo.PlatformType(parsed.Scheme),
137+
Type: repository.RepositoryType(parsed.Scheme),
140138
Path: path,
141139
}
142140
}

internal/config/patrol_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
package config
22

33
import (
4-
"sheriff/internal/repo"
4+
"sheriff/internal/repository"
55
"testing"
66

77
"github.com/stretchr/testify/assert"
88
)
99

1010
func TestGetPatrolConfiguration(t *testing.T) {
1111
want := PatrolConfig{
12-
Locations: []ProjectLocation{{Type: repo.Gitlab, Path: "group1"}, {Type: repo.Gitlab, Path: "group2/project1"}},
12+
Locations: []ProjectLocation{{Type: repository.Gitlab, Path: "group1"}, {Type: repository.Gitlab, Path: "group2/project1"}},
1313
ReportToEmails: []string{"[email protected]"},
1414
ReportToSlackChannels: []string{"report-slack-channel"},
1515
ReportToIssue: true,
@@ -29,7 +29,7 @@ func TestGetPatrolConfiguration(t *testing.T) {
2929

3030
func TestGetPatrolConfigurationCLIOverridesFile(t *testing.T) {
3131
want := PatrolConfig{
32-
Locations: []ProjectLocation{{Type: repo.Gitlab, Path: "group1"}, {Type: repo.Gitlab, Path: "group2/project1"}},
32+
Locations: []ProjectLocation{{Type: repository.Gitlab, Path: "group1"}, {Type: repository.Gitlab, Path: "group2/project1"}},
3333
ReportToEmails: []string{"[email protected]", "[email protected]"},
3434
ReportToSlackChannels: []string{"other-slack-channel"},
3535
ReportToIssue: false,
@@ -87,8 +87,8 @@ func TestParseUrls(t *testing.T) {
8787
{[]string{"gitlab://namespace/project"}, &ProjectLocation{Type: "gitlab", Path: "namespace/project"}, false},
8888
{[]string{"gitlab://namespace/subgroup/project"}, &ProjectLocation{Type: "gitlab", Path: "namespace/subgroup/project"}, false},
8989
{[]string{"gitlab://namespace"}, &ProjectLocation{Type: "gitlab", Path: "namespace"}, false},
90-
{[]string{"github://organization"}, &ProjectLocation{Type: "github", Path: "organization"}, true},
91-
{[]string{"github://organization/project"}, &ProjectLocation{Type: "github", Path: "organization/project"}, true},
90+
{[]string{"github://organization"}, &ProjectLocation{Type: "github", Path: "organization"}, false},
91+
{[]string{"github://organization/project"}, &ProjectLocation{Type: "github", Path: "organization/project"}, false},
9292
{[]string{"unknown://namespace/project"}, nil, true},
9393
{[]string{"unknown://not a path"}, nil, true},
9494
{[]string{"not a target"}, nil, true},

internal/git/client.go

Lines changed: 0 additions & 19 deletions
This file was deleted.

internal/git/git.go

Lines changed: 0 additions & 35 deletions
This file was deleted.

internal/git/git_test.go

Lines changed: 0 additions & 48 deletions
This file was deleted.

internal/patrol/patrol.go

Lines changed: 48 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ import (
77
"fmt"
88
"os"
99
"sheriff/internal/config"
10-
"sheriff/internal/git"
1110
"sheriff/internal/publish"
12-
"sheriff/internal/repo"
11+
"sheriff/internal/repository"
12+
"sheriff/internal/repository/provider"
1313
"sheriff/internal/scanner"
1414
"sheriff/internal/slack"
1515
"sync"
@@ -29,21 +29,19 @@ type securityPatroller interface {
2929

3030
// sheriffService is the implementation of the SecurityPatroller interface.
3131
type sheriffService struct {
32-
gitlabService repo.IService
33-
slackService slack.IService
34-
gitService git.IService
35-
osvService scanner.VulnScanner[scanner.OsvReport]
32+
repoService provider.IProvider
33+
slackService slack.IService
34+
osvService scanner.VulnScanner[scanner.OsvReport]
3635
}
3736

3837
// New creates a new securityPatroller service.
3938
// It contains the main "loop" logic of this tool.
4039
// A "patrol" is defined as scanning GitLab groups for vulnerabilities and publishing reports where needed.
41-
func New(gitlabService repo.IService, slackService slack.IService, gitService git.IService, osvService scanner.VulnScanner[scanner.OsvReport]) securityPatroller {
40+
func New(repoService provider.IProvider, slackService slack.IService, osvService scanner.VulnScanner[scanner.OsvReport]) securityPatroller {
4241
return &sheriffService{
43-
gitlabService: gitlabService,
44-
slackService: slackService,
45-
gitService: gitService,
46-
osvService: osvService,
42+
repoService: repoService,
43+
slackService: slackService,
44+
osvService: osvService,
4745
}
4846
}
4947

@@ -65,7 +63,7 @@ func (s *sheriffService) Patrol(args config.PatrolConfig) (warn error, err error
6563

6664
if args.ReportToIssue {
6765
log.Info().Msg("Creating issue in affected projects")
68-
if gwarn := publish.PublishAsGitlabIssues(scanReports, s.gitlabService); gwarn != nil {
66+
if gwarn := publish.PublishAsIssues(scanReports, s.repoService); gwarn != nil {
6967
gwarn = errors.Join(errors.New("errors occured when creating issues"), gwarn)
7068
warn = errors.Join(gwarn, warn)
7169
}
@@ -107,13 +105,7 @@ func (s *sheriffService) scanAndGetReports(locations []config.ProjectLocation) (
107105
defer os.RemoveAll(tempScanDir)
108106
log.Info().Str("path", tempScanDir).Msg("Created temporary directory")
109107

110-
gitlabLocs := pie.Map(
111-
pie.Filter(locations, func(v config.ProjectLocation) bool { return v.Type == repo.Gitlab }),
112-
func(v config.ProjectLocation) string { return v.Path },
113-
)
114-
log.Info().Strs("locations", gitlabLocs).Msg("Getting the list of projects to scan")
115-
116-
projects, pwarn := s.gitlabService.GetProjectList(gitlabLocs)
108+
projects, pwarn := s.getProjectList(locations)
117109
if pwarn != nil {
118110
pwarn = errors.Join(errors.New("errors occured when getting project list"), pwarn)
119111
warn = errors.Join(pwarn, warn)
@@ -152,18 +144,51 @@ func (s *sheriffService) scanAndGetReports(locations []config.ProjectLocation) (
152144
return
153145
}
154146

147+
func (s *sheriffService) getProjectList(locs []config.ProjectLocation) (projects []repository.Project, warn error) {
148+
gitlabLocs := pie.Map(
149+
pie.Filter(locs, func(loc config.ProjectLocation) bool { return loc.Type == repository.Gitlab }),
150+
func(loc config.ProjectLocation) string { return loc.Path },
151+
)
152+
githubLocs := pie.Map(
153+
pie.Filter(locs, func(loc config.ProjectLocation) bool { return loc.Type == repository.Github }),
154+
func(loc config.ProjectLocation) string { return loc.Path },
155+
)
156+
157+
if len(gitlabLocs) > 0 {
158+
log.Info().Strs("locations", gitlabLocs).Msg("Getting the list of projects from gitlab to scan")
159+
gitlabProjects, err := s.repoService.Provide(repository.Gitlab).GetProjectList(gitlabLocs)
160+
if err != nil {
161+
warn = errors.Join(errors.New("non-critical errors encountered when scanning for gitlab projects"), err)
162+
}
163+
164+
projects = append(projects, gitlabProjects...)
165+
}
166+
167+
if len(githubLocs) > 0 {
168+
log.Info().Strs("locations", githubLocs).Msg("Getting the list of projects from github to scan")
169+
githubProjects, err := s.repoService.Provide(repository.Github).GetProjectList(githubLocs)
170+
if err != nil {
171+
warn = errors.Join(errors.New("non-critical errors encountered when scanning for github projects"), err)
172+
}
173+
174+
projects = append(projects, githubProjects...)
175+
}
176+
177+
return
178+
}
179+
155180
// scanProject scans a project for vulnerabilities using the osv scanner.
156-
func (s *sheriffService) scanProject(project repo.Project) (report *scanner.Report, err error) {
181+
func (s *sheriffService) scanProject(project repository.Project) (report *scanner.Report, err error) {
157182
dir, err := os.MkdirTemp(tempScanDir, fmt.Sprintf("%v-", project.Name))
158183
if err != nil {
159184
return nil, errors.Join(errors.New("failed to create project temporary directory"), err)
160185
}
161186
defer os.RemoveAll(dir)
162187

163188
// Clone the project
164-
log.Info().Str("project", project.Path).Str("dir", dir).Msg("Cloning project")
165-
if err = s.gitService.Clone(dir, project.RepoUrl); err != nil {
166-
return nil, errors.Join(errors.New("failed to clone project"), err)
189+
log.Info().Str("project", project.Path).Str("dir", dir).Str("url", project.RepoUrl).Msg("Cloning project")
190+
if err := s.repoService.Provide(project.Repository).Clone(project.RepoUrl, dir); err != nil {
191+
return nil, errors.Join(fmt.Errorf("failed to clone project %v", project.Path), err)
167192
}
168193

169194
config := config.GetProjectConfiguration(project.Path, dir)

0 commit comments

Comments
 (0)