diff --git a/.2ms.yml b/.2ms.yml index 2c13a979..09d517cc 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -51,3 +51,5 @@ ignore-result: - 51a6f4e3c7e3a79c9722abb7541b4902098e526b # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 - 53803ee7e880952e926898a434acff4483fec67e # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 - aa52405f239a8be1284d933025c557b071b24036 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 + - 854547fc6e35c0d1f63c0f4d426aebd4d64679fc # False positive, see https://github.com/gitleaks/gitleaks/pull/1358, found at https://github.com/Checkmarx/2ms/commit/45a5c9d35ff910dfec5e5a76cdedb8977da5dd34#diff-d712d2256df359061d691b711ca7ed30ba408199b1e3801cef289779778d8bad + - b7c3ac03d8a24892a2c4be5810ce73ffdf6ba3ae # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 diff --git a/cmd/workers_test.go b/cmd/workers_test.go index 8152ae20..f3f35129 100644 --- a/cmd/workers_test.go +++ b/cmd/workers_test.go @@ -30,6 +30,10 @@ func (i *mockItem) GetSource() string { return i.source } +func (i *mockItem) GetGitInfo() *plugins.GitInfo { + return nil +} + func TestProcessItems(t *testing.T) { totalItemsToProcess := 5 engineConfig := engine.EngineConfig{} diff --git a/engine/engine.go b/engine/engine.go index a0581b57..53f2fb59 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -85,12 +85,20 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S Raw: *item.GetContent(), FilePath: item.GetSource(), } + gitInfo := item.GetGitInfo() for _, value := range e.detector.Detect(fragment) { itemId := getFindingId(item, value) var startLine, endLine int + var err error if pluginName == "filesystem" { startLine = value.StartLine + 1 endLine = value.EndLine + 1 + } else if pluginName == "git" { + startLine, endLine, err = plugins.GetGitStartAndEndLine(gitInfo, value.StartLine, value.EndLine) + if err != nil { + errors <- fmt.Errorf("failed to get git lines for source %s: %w", item.GetSource(), err) + return + } } else { startLine = value.StartLine endLine = value.EndLine diff --git a/engine/engine_test.go b/engine/engine_test.go index d31fa39e..8fb67a96 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -195,3 +195,7 @@ func (i item) GetSource() string { } return "test" } + +func (i item) GetGitInfo() *plugins.GitInfo { + return nil +} diff --git a/go.mod b/go.mod index 35524222..4fcf8c29 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( github.com/rs/zerolog v1.32.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.8.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e - github.com/stretchr/testify v1.9.0 + github.com/spf13/pflag v1.0.6 + github.com/spf13/viper v1.20.1 + github.com/stretchr/testify v1.10.0 github.com/zricethezav/gitleaks/v8 v8.18.2 golang.org/x/time v0.5.0 gopkg.in/yaml.v3 v3.0.1 @@ -23,36 +23,31 @@ require ( github.com/charmbracelet/lipgloss v0.7.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/semgroup v1.2.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/h2non/filetype v1.1.3 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/cast v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.35.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/sync v0.11.0 // indirect + golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/text v0.23.0 // indirect ) //transitive dependency not applied but also reported as not used by go mod why diff --git a/go.sum b/go.sum index b79fd945..8e77827a 100644 --- a/go.sum +++ b/go.sum @@ -8,7 +8,6 @@ github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZ github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -16,12 +15,14 @@ github.com/fatih/semgroup v1.2.0 h1:h/OLXwEM+3NNyAdZEpMiH1OzfplU09i2qXPVThGZvyg= github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS879kgm8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg= github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -31,8 +32,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -43,8 +42,6 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb h1:w1g9wNDIE/pHSTmAaUhv4TZQuPBS6GV3mMz5hkgziIU= github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -54,14 +51,12 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= -github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= -github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -76,34 +71,26 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= +github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ= github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e h1:NghP/tGibNy35kVZJjwUR8HjoHOW1TUucmO/DUikdRs= -github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e/go.mod h1:Hqr8J4/Q1O00v/4zIIggDIidAoD4w8Oqtzc+Ew8QO+I= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -116,8 +103,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -129,8 +114,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -148,8 +133,8 @@ golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -162,8 +147,5 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/scan.go b/pkg/scan.go index 9300382e..27899eb1 100644 --- a/pkg/scan.go +++ b/pkg/scan.go @@ -91,7 +91,6 @@ func (s *scanner) Scan(scanItems []ScanItem, scanConfig ScanConfig) (*reporting. func (s *scanner) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (*reporting.Report, error) { itemsCh := cmd.Channels.Items errorsCh := cmd.Channels.Errors - wg := &sync.WaitGroup{} // Initialize engine configuration. engineConfig := engine.EngineConfig{IgnoredIds: scanConfig.IgnoreResultIds} @@ -113,18 +112,10 @@ func (s *scanner) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (* cmd.Channels.WaitGroup.Add(1) go cmd.ProcessScoreWithoutValidation(engineInstance) - // Forward scan items from itemsIn to itemsCh. - go func() { - for item := range itemsIn { - wg.Add(1) - go func(i ScanItem) { - defer wg.Done() - itemsCh <- i - }(item) - } - wg.Wait() - close(itemsCh) - }() + for item := range itemsIn { + itemsCh <- item + } + close(itemsCh) // Wait for all processing routines. cmd.Channels.WaitGroup.Wait() diff --git a/pkg/scanner.go b/pkg/scanner.go index 43452fa8..9c273781 100644 --- a/pkg/scanner.go +++ b/pkg/scanner.go @@ -27,6 +27,10 @@ func (i ScanItem) GetSource() string { return i.Source } +func (i ScanItem) GetGitInfo() *plugins.GitInfo { + return nil +} + type Scanner interface { Scan(scanItems []ScanItem, scanConfig ScanConfig) (*reporting.Report, error) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (*reporting.Report, error) diff --git a/plugins/git.go b/plugins/git.go index 62b16e79..9e0bede8 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -13,10 +13,18 @@ import ( git "github.com/zricethezav/gitleaks/v8/sources" ) +type DiffType int + +const ( + AddedContent DiffType = iota + RemovedContent +) + const ( argDepth = "depth" argScanAllBranches = "all-branches" argProjectName = "project-name" + unknownCommit = "unknown" ) type GitPlugin struct { @@ -27,6 +35,11 @@ type GitPlugin struct { projectName string } +type GitInfo struct { + Hunks []*gitdiff.TextFragment + ContentType DiffType +} + func (p *GitPlugin) GetName() string { return "git" } @@ -69,36 +82,91 @@ func (p *GitPlugin) buildScanOptions() string { } func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan ISourceItem, errChan chan error) { - diffs, close := p.readGitLog(path, scanOptions, errChan) - defer close() + diffs, wait := p.readGitLog(path, scanOptions, errChan) + defer wait() for file := range diffs { - if file.PatchHeader == nil { - // While parsing the PatchHeader, the token size limit may be exceeded, resulting in a nil value. - // This scenario is unlikely, but it causes the scan to never complete. - file.PatchHeader = &gitdiff.PatchHeader{} + p.processFileDiff(file, itemsChan) + } +} + +// processFileDiff handles processing a single diff file. +func (p *GitPlugin) processFileDiff(file *gitdiff.File, itemsChan chan ISourceItem) { + if file.PatchHeader == nil { + // When parsing the PatchHeader, the token size limit may be exceeded, resulting in a nil value + // This scenario is unlikely but may cause the scan to never complete + file.PatchHeader = &gitdiff.PatchHeader{} + file.PatchHeader.SHA = unknownCommit + } + + log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) + + // Skip binary files + if file.IsBinary { + return + } + + // Extract the changes (added and removed) from the text fragments + addedChanges, removedChanges := extractChanges(file.TextFragments) + + var fileName string + if file.IsDelete { + fileName = file.OldName + } else { + fileName = file.NewName + } + id := fmt.Sprintf("%s-%s-%s-%s", p.GetName(), p.projectName, file.PatchHeader.SHA, fileName) + source := fmt.Sprintf("git show %s:%s", file.PatchHeader.SHA, fileName) + + // If there are added changes, send an item with added content + if addedChanges != "" { + itemsChan <- item{ + Content: &addedChanges, + ID: id, + Source: source, + GitInfo: &GitInfo{ + Hunks: file.TextFragments, + ContentType: AddedContent, + }, } + } - log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) - if file.IsBinary || file.IsDelete { - continue + // If there are removed changes, send an item with removed content + if removedChanges != "" { + itemsChan <- item{ + Content: &removedChanges, + ID: id, + Source: source, + GitInfo: &GitInfo{ + Hunks: file.TextFragments, + ContentType: RemovedContent, + }, } + } +} - fileChanges := "" - for _, textFragment := range file.TextFragments { - if textFragment != nil { - raw := textFragment.Raw(gitdiff.OpAdd) - fileChanges += raw - } +// extractChanges iterates over the text fragments to compile added and removed changes +func extractChanges(fragments []*gitdiff.TextFragment) (added string, removed string) { + var addedBuilder, removedBuilder strings.Builder + + for _, tf := range fragments { + if tf == nil { + continue } - if fileChanges != "" { - itemsChan <- item{ - Content: &fileChanges, - ID: fmt.Sprintf("%s-%s-%s-%s", p.GetName(), p.projectName, file.PatchHeader.SHA, file.NewName), - Source: fmt.Sprintf("git show %s:%s", file.PatchHeader.SHA, file.NewName), + for i := range tf.Lines { + switch tf.Lines[i].Op { + case gitdiff.OpAdd: + addedBuilder.WriteString(tf.Lines[i].Line) + case gitdiff.OpDelete: + removedBuilder.WriteString(tf.Lines[i].Line) + default: } + // Clean up the line content to free memory + tf.Lines[i].Line = "" } } + + return addedBuilder.String(), removedBuilder.String() } func (p *GitPlugin) readGitLog(path string, scanOptions string, errChan chan error) (<-chan *gitdiff.File, func()) { @@ -138,3 +206,57 @@ func validGitRepoArgs(cmd *cobra.Command, args []string) error { } return nil } + +// GetGitStartAndEndLine walks the diff hunks and discover the actual start and end lines of the file +func GetGitStartAndEndLine(gitInfo *GitInfo, localStartLine, localEndLine int) (int, int, error) { + hunkPosition, hunkCount, relevantOp, err := getHunkPosAndCount(gitInfo) + if err != nil { + return 0, 0, fmt.Errorf("failed to get hunk position and count: %w", err) + } + + diffLines := 0 // Tracks how many lines have been processed in the diff + for _, hunk := range gitInfo.Hunks { + // If the hunk ends before the start line in the file diff, skip it + totalLines := hunkCount(hunk) + if diffLines+totalLines <= localStartLine { + diffLines += totalLines + continue + } + + // Get the start line of the hunk in the file diff and walk through its lines to find the actual start line + fileStartLine := hunkPosition(hunk) - 1 + for _, line := range hunk.Lines { + switch line.Op { + case relevantOp: + fileStartLine += 1 + if diffLines == localStartLine { + fileEndLine := fileStartLine + (localEndLine - localStartLine) + return fileStartLine, fileEndLine, nil + } + diffLines += 1 + case gitdiff.OpContext: // Context lines are not counted in the diff + fileStartLine += 1 + default: + } + } + } + // Did not find the start line in any hunk + return 0, 0, fmt.Errorf("failed to find start line %d in hunks", localStartLine) +} + +// getHunkPosAndCount returns the functions to get the position and count of hunks based on the content type +func getHunkPosAndCount(gitInfo *GitInfo) (hunkPos func(h *gitdiff.TextFragment) int, hunkCount func(h *gitdiff.TextFragment) int, matchOp gitdiff.LineOp, err error) { + switch gitInfo.ContentType { + case AddedContent: + hunkPos = func(h *gitdiff.TextFragment) int { return int(h.NewPosition) } + hunkCount = func(h *gitdiff.TextFragment) int { return int(h.LinesAdded) } + matchOp = gitdiff.OpAdd + case RemovedContent: + hunkPos = func(h *gitdiff.TextFragment) int { return int(h.OldPosition) } + hunkCount = func(h *gitdiff.TextFragment) int { return int(h.LinesDeleted) } + matchOp = gitdiff.OpDelete + default: + err = fmt.Errorf("unknown content type: %d", gitInfo.ContentType) + } + return +} diff --git a/plugins/git_test.go b/plugins/git_test.go index 85ac71cf..8a71aff7 100644 --- a/plugins/git_test.go +++ b/plugins/git_test.go @@ -3,6 +3,7 @@ package plugins import ( "errors" "fmt" + "github.com/gitleaks/go-gitdiff/gitdiff" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "os" @@ -164,3 +165,250 @@ func TestValidGitRepoArgs(t *testing.T) { }) } } + +func TestGetGitStartAndEndLine(t *testing.T) { + tests := []struct { + name string + gitInfo *GitInfo + localStartLine int + localEndLine int + expectedFileStartLine int + expectedFileEndLine int + }{ + { + name: "Secret in added content without context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(9, 0, 10, 3, 3, 0, nil), + createMockHunk(49, 0, 53, 3, 3, 0, nil), + createMockHunk(55, 0, 62, 2, 2, 0, nil), + createMockHunk(58, 0, 67, 1, 1, 0, nil), + createMockHunk(103, 11, 112, 1, 1, 11, nil), + }, + ContentType: AddedContent, + }, + localStartLine: 9, + localEndLine: 9, + expectedFileStartLine: 112, + expectedFileEndLine: 112, + }, + { + name: "Secret in removed content without context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(10, 2, 10, 1, 1, 2, nil), + createMockHunk(29, 0, 29, 1, 1, 0, nil), + createMockHunk(46, 8, 46, 1, 1, 8, nil), + createMockHunk(57, 2, 50, 1, 1, 2, nil), + createMockHunk(63, 2, 55, 2, 2, 2, nil), + createMockHunk(106, 0, 99, 1, 1, 0, nil), + createMockHunk(108, 8, 101, 3, 3, 8, nil), + }, + ContentType: RemovedContent, + }, + localStartLine: 18, + localEndLine: 18, + expectedFileStartLine: 112, + expectedFileEndLine: 112, + }, + { + name: "Secret in added content with context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(7, 8, 7, 7, 1, 2, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + createMockHunk(27, 6, 26, 7, 1, 0, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + }, + ContentType: AddedContent, + }, + localStartLine: 1, + localEndLine: 1, + expectedFileStartLine: 29, + expectedFileEndLine: 29, + }, + { + name: "Secret in removed content with context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(475, 8, 475, 8, 2, 2, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + createMockHunk(512, 8, 512, 8, 2, 2, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + }, + ContentType: RemovedContent, + }, + localStartLine: 3, + localEndLine: 3, + expectedFileStartLine: 516, + expectedFileEndLine: 516, + }, + { + name: "validate skip hunk when secret is found immediately after the hunk before in added content", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(975, 0, 976, 3, 3, 0, nil), + createMockHunk(977, 4, 980, 1, 1, 4, nil), + }, + ContentType: AddedContent, + }, + localStartLine: 3, + localEndLine: 3, + expectedFileStartLine: 980, + expectedFileEndLine: 980, + }, + { + name: "validate skip hunk when secret is found immediately after the hunk before in removed content", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(976, 3, 975, 0, 0, 3, nil), + createMockHunk(980, 1, 977, 4, 4, 1, nil), + }, + ContentType: RemovedContent, + }, + localStartLine: 3, + localEndLine: 3, + expectedFileStartLine: 980, + expectedFileEndLine: 980, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actualFileStartLine, actualFileEndLine, err := GetGitStartAndEndLine(tt.gitInfo, tt.localStartLine, tt.localEndLine) + if err != nil { + t.Fatalf("GetGitStartAndEndLine() error = %v", err) + } + assert.Equal(t, tt.expectedFileStartLine, actualFileStartLine) + assert.Equal(t, tt.expectedFileEndLine, actualFileEndLine) + }) + } +} + +func createMockHunk(oldPos, oldLines, newPos, newLines, linesAdded, linesDeleted int64, lines []gitdiff.Line) *gitdiff.TextFragment { + if lines == nil { + for i := int64(0); i < linesDeleted; i++ { + lines = append(lines, gitdiff.Line{Op: gitdiff.OpDelete}) + } + for i := int64(0); i < linesAdded; i++ { + lines = append(lines, gitdiff.Line{Op: gitdiff.OpAdd}) + } + } + return &gitdiff.TextFragment{ + OldPosition: oldPos, + OldLines: oldLines, + NewPosition: newPos, + NewLines: newLines, + LinesAdded: linesAdded, + LinesDeleted: linesDeleted, + Lines: lines, + } +} diff --git a/plugins/plugins.go b/plugins/plugins.go index 9777b0b2..b78ff249 100644 --- a/plugins/plugins.go +++ b/plugins/plugins.go @@ -10,6 +10,7 @@ type ISourceItem interface { GetContent() *string GetID() string GetSource() string + GetGitInfo() *GitInfo } type item struct { @@ -17,7 +18,8 @@ type item struct { // Unique identifier of the item ID string // User friendly description and/or link to the item - Source string + Source string + GitInfo *GitInfo } var _ ISourceItem = (*item)(nil) @@ -34,6 +36,10 @@ func (i item) GetSource() string { return i.Source } +func (i item) GetGitInfo() *GitInfo { + return i.GitInfo +} + type Plugin struct { ID string Limit chan struct{}