Use .Line to reduce diff strangeness (#380)

Author: bill-rich

main.go

@@ -33,6 +33,7 @@ var (
 	cli            = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
 	cmd            string
 	debug          = cli.Flag("debug", "Run in debug mode.").Bool()
+	trace          = cli.Flag("trace", "Run in trace mode.").Bool()
 	jsonOut        = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy     = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	concurrency    = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
@@ -96,10 +97,14 @@ func init() {
 	if *jsonOut {
 		logrus.SetFormatter(&logrus.JSONFormatter{})
 	}
-	if *debug {
+	switch {
+	case *trace:
+		logrus.SetLevel(logrus.TraceLevel)
+		logrus.Debugf("running version %s", version.BuildVersion)
+	case *debug:
 		logrus.SetLevel(logrus.DebugLevel)
 		logrus.Debugf("running version %s", version.BuildVersion)
-	} else {
+	default:
 		logrus.SetLevel(logrus.InfoLevel)
 	}
 }

pkg/sources/git/git.go

@@ -278,6 +278,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
 
 	var depth int64
 	for file := range fileChan {
+		log.WithField("commit", file.PatchHeader.SHA).WithField("file", file.NewName).Trace("Scanning file from git")
 		if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
 			log.Debugf("reached max depth")
 			break
@@ -311,9 +312,10 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
 			newLineNumber := frag.NewPosition
 			for _, line := range frag.Lines {
 				if line.Op == gitdiff.OpAdd {
-					newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n")
+					newLines.WriteString(line.Line)
 				}
 			}
+			log.WithField("fragment", newLines.String()).Trace("detecting fragment")
 			metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, newLineNumber)
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,

pkg/sources/git/git_test.go

@@ -169,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) {
 				},
 			},
 			expectedChunkData: map[string]*byteCompare{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826-aws":   {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")},
-				"a6f8aa55736d4a85be31a0048a4607396898647a-bump":  {B: []byte("+f \n")},
-				"07d96d011005fe8296bdd237c13a06a72e96783d-bump":  {B: []byte("+ s  \n")},
-				"2f251b8c1e72135a375b659951097ec7749d4af9-bump":  {B: []byte("+  \n")},
-				"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump":  {B: []byte("+oops  \n")},
-				"735b52b0eb40610002bb1088e902bd61824eb305-bump":  {B: []byte("+oops \n")},
-				"ce62d79908803153ef6e145e042d3e80488ef747-bump":  {B: []byte("+ \n")},
-				"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")},
-				"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true},
-				"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws":   {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true},
-				"90c75f884c65dc3638ca1610bd9844e668f213c2-aws":   {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826-aws":   {B: []byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")},
+				"a6f8aa55736d4a85be31a0048a4607396898647a-bump":  {B: []byte("f\n")},
+				"07d96d011005fe8296bdd237c13a06a72e96783d-bump":  {B: []byte(" s \n")},
+				"2f251b8c1e72135a375b659951097ec7749d4af9-bump":  {B: []byte(" \n")},
+				"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump":  {B: []byte("oops \n")},
+				"735b52b0eb40610002bb1088e902bd61824eb305-bump":  {B: []byte("oops\n")},
+				"ce62d79908803153ef6e145e042d3e80488ef747-bump":  {B: []byte("\n")},
+				"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!")},
+				"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that"), Multi: true},
+				"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws":   {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye"), Multi: true},
+				"90c75f884c65dc3638ca1610bd9844e668f213c2-aws":   {B: []byte("this is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n"), Multi: true},
 			},
 		},
 	}