add ability to change annotation level (#72)

* add ability to change annotation level

* fix readme json

* test violation

* When warn annotation is set, do not fail check

* Add log warning to unknown annotation level

* re-order log to make sure to capture a useful message

* Correct log levels in circle ci logging

* add test to run binary

* pass through new config in circleci

* Revert "test violation"

This reverts commit 395d23d.

Author: Dan Hertz

.circleci/config.yml

@@ -21,6 +21,17 @@ jobs:
           name: Clean up reusable Docker
           command: clean_up_reusable_docker
 
+  run-local:
+    docker:
+      - image: cimg/go:1.18.4
+    steps:
+      - checkout
+      - run:
+          name: run binary locally
+          command: make
+          environment:
+            GITHUB_WORKSPACE: "."
+
   lint:
     docker:
       - image: deliveroo/circleci:0.2.8
@@ -83,6 +94,11 @@ workflows:
             tags:
               only: /.*/
           context: build
+      - run-local:
+          filters:
+            tags:
+              only: /.*/
+          context: build
       - lint:
           filters:
             tags:

Makefile

@@ -25,7 +25,7 @@ clean:
 	$(GOCLEAN)
 	rm -f $(BINARY_NAME)
 start:
-	./$(BINARY_NAME)
+	$(BINARY_NAME)
 dockertag:
 	docker tag $(NAME):latest $(VERSION)
 	docker tag $(NAME):latest $(LATEST)

README.md

@@ -45,7 +45,8 @@ config will be equivalent to the snippet below:
         }
       ]
     }
-  ]
+  ],
+  "annotationLevel": "failure"
 }
 ```
 
@@ -152,7 +153,12 @@ object:
 If your config file specifies a non-null value for `defaultRedactionConfig`, then exactly one of the above keys must be
 filled out; if more than one is filled out, the Nightfall API will return a 400 error code.
 
-For more information on how to configure redaction-related fields, refer to the [Nightfall docs](https://docs.nightfall.ai/reference/scanpayloadv3). 
+For more information on how to configure redaction-related fields, refer to the [Nightfall docs](https://docs.nightfall.ai/reference/scanpayloadv3).
+
+### Annotation Level customization
+
+Annotations can be configured to be `notice`, `warning`, or `failure`, by setting the `annotationLevel` key in the
+configuration object. The check will only fail if `failure` annotations are written.
 
 ## Configuration Examples
 
@@ -332,3 +338,9 @@ Encryption:
   }
 }
 ```
+
+Annotate as Warnings:
+```json
+{
+  "annotationLevel": "warning"
+}

cmd/nightfalldlp/main.go

@@ -60,7 +60,7 @@ func run() error {
 		return err
 	}
 
-	return diffReviewClient.WriteComments(comments)
+	return diffReviewClient.WriteComments(comments, nightfallConfig.AnnotationLevel)
 }
 
 // usingGithubAction determine if nightfalldlp is being run by

go.sum

@@ -172,6 +172,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -287,6 +288,7 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/clients/diffreviewer/circleci/circleci_service.go

@@ -120,6 +120,7 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
 		FileInclusionList:           nightfallConfig.FileInclusionList,
 		FileExclusionList:           nightfallConfig.FileExclusionList,
 		DefaultRedactionConfig:      nightfallConfig.DefaultRedactionConfig,
+		AnnotationLevel:             nightfallConfig.AnnotationLevel,
 	}, nil
 }
 
@@ -198,14 +199,18 @@ func (s *Service) GetDiff() ([]*diffreviewer.FileDiff, error) {
 }
 
 // WriteComments posts the findings as annotations to the github check
-func (s *Service) WriteComments(comments []*diffreviewer.Comment) error {
+func (s *Service) WriteComments(comments []*diffreviewer.Comment, level string) error {
 	if len(comments) == 0 {
 		s.Logger.Info("no sensitive items found")
 		return nil
 	}
-	s.logCommentsToCircle(comments)
+	s.logCommentsToCircle(comments, level)
+	var returnErr error
+	if level == nightfallconfig.AnnotationLevelFailure {
+		returnErr = errSensitiveItemsFound
+	}
 	if s.GithubClient == nil {
-		return errSensitiveItemsFound
+		return returnErr
 	}
 	if s.PrDetails.PrNumber != nil {
 		existingComments, _, err := s.GithubClient.PullRequestsService().ListComments(
@@ -218,7 +223,7 @@ func (s *Service) WriteComments(comments []*diffreviewer.Comment) error {
 		if err != nil {
 			s.Logger.Error(fmt.Sprintf("Error listing existing pull request comments: %s", err.Error()))
 		}
-		githubComments := s.createGithubPullRequestComments(comments)
+		githubComments := s.createGithubPullRequestComments(comments, level)
 		filteredGithubComments := filterExistingComments(githubComments, existingComments)
 		for _, c := range filteredGithubComments {
 			_, _, err := s.GithubClient.PullRequestsService().CreateComment(
@@ -233,7 +238,7 @@ func (s *Service) WriteComments(comments []*diffreviewer.Comment) error {
 			}
 		}
 	} else {
-		githubComments := s.createGithubRepositoryComments(comments)
+		githubComments := s.createGithubRepositoryComments(comments, level)
 		for _, c := range githubComments {
 			_, _, err := s.GithubClient.RepositoriesService().CreateComment(
 				context.Background(),
@@ -248,26 +253,37 @@ func (s *Service) WriteComments(comments []*diffreviewer.Comment) error {
 		}
 	}
 	// returning error to fail circleCI step
-	return errSensitiveItemsFound
+	return returnErr
 }
 
-func (s *Service) logCommentsToCircle(comments []*diffreviewer.Comment) {
+func (s *Service) logCommentsToCircle(comments []*diffreviewer.Comment, level string) {
 	for _, comment := range comments {
-		s.Logger.Error(fmt.Sprintf(
+		logString := fmt.Sprintf(
 			"%s at %s on line %d",
 			comment.Body,
 			comment.FilePath,
 			comment.LineNumber,
-		))
+		)
+		switch level {
+		case nightfallconfig.AnnotationLevelFailure:
+			s.Logger.Error(logString)
+		case nightfallconfig.AnnotationLevelWarning:
+			s.Logger.Warning(logString)
+		case nightfallconfig.AnnotationLevelNotice:
+			s.Logger.Info(logString)
+		default:
+			s.Logger.Error(logString)
+		}
 	}
 }
 
-func (s *Service) createGithubPullRequestComments(comments []*diffreviewer.Comment) []*github.PullRequestComment {
+func (s *Service) createGithubPullRequestComments(comments []*diffreviewer.Comment, level string) []*github.PullRequestComment {
 	githubComments := make([]*github.PullRequestComment, len(comments))
 	for i, comment := range comments {
+		body := fmt.Sprintf("%s: %s", level, comment.Body)
 		githubComments[i] = &github.PullRequestComment{
 			CommitID: &s.PrDetails.CommitSha,
-			Body:     &comment.Body,
+			Body:     &body,
 			Path:     &comment.FilePath,
 			Line:     &comment.LineNumber,
 			Side:     github.String(GithubCommentRightSide),
@@ -276,12 +292,13 @@ func (s *Service) createGithubPullRequestComments(comments []*diffreviewer.Comme
 	return githubComments
 }
 
-func (s *Service) createGithubRepositoryComments(comments []*diffreviewer.Comment) []*github.RepositoryComment {
+func (s *Service) createGithubRepositoryComments(comments []*diffreviewer.Comment, level string) []*github.RepositoryComment {
 	githubComments := make([]*github.RepositoryComment, len(comments))
 	for i, comment := range comments {
+		body := fmt.Sprintf("%s: %s", level, comment.Body)
 		githubComments[i] = &github.RepositoryComment{
 			CommitID: &s.PrDetails.CommitSha,
-			Body:     &comment.Body,
+			Body:     &body,
 			Path:     &comment.FilePath,
 			Position: &comment.LineNumber,
 		}

internal/clients/diffreviewer/circleci/circleci_service_test.go

@@ -216,6 +216,7 @@ func (c *circleCiTestSuite) TestLoadConfig() {
 		DefaultRedactionConfig: &nf.RedactionConfig{
 			SubstitutionConfig: &nf.SubstitutionConfig{SubstitutionPhrase: "REDACTED"},
 		},
+		AnnotationLevel: "warning",
 	}
 
 	nightfallConfig, err := tp.cs.LoadConfig(testConfigFileName)
@@ -246,6 +247,7 @@ func (c *circleCiTestSuite) TestLoadConfigDetectionRuleUUID() {
 		TokenExclusionList:          []string{excludedCreditCardRegex, excludedApiToken, excludedIPRegex},
 		FileInclusionList:           []string{"*"},
 		FileExclusionList:           []string{".nightfalldlp/config.json"},
+		AnnotationLevel:             "failure",
 	}
 
 	nightfallConfig, err := tp.cs.LoadConfig(testConfigDetectionRuleUUIDFileName)
@@ -326,6 +328,7 @@ func (c *circleCiTestSuite) TestLoadEmptyConfig() {
 				NumCharsToLeaveUnmasked: 2,
 			},
 		},
+		AnnotationLevel: "failure",
 	}
 
 	nightfallConfig, err := tp.cs.LoadConfig(testEmptyConfigFileName)
@@ -367,6 +370,7 @@ func (c *circleCiTestSuite) TestWriteCircleComments() {
 		"/comments.txt",
 		tp.cs.PrDetails.CommitSha,
 		60,
+		"notice",
 	)
 	emptyComments := make([]*diffreviewer.Comment, 0)
 
@@ -377,8 +381,8 @@ func (c *circleCiTestSuite) TestWriteCircleComments() {
 	}{
 		{
 			giveComments: testComments,
-			wantErr:      errSensitiveItemsFound,
-			desc:         "single batch comments test",
+			wantErr:      nil,
+			desc:         "notice: single batch comments test",
 		},
 		{
 			giveComments: emptyComments,
@@ -392,15 +396,15 @@ func (c *circleCiTestSuite) TestWriteCircleComments() {
 			mockLogger.EXPECT().Info("no sensitive items found")
 		}
 		for _, comment := range tt.giveComments {
-			mockLogger.EXPECT().Error(fmt.Sprintf(
+			mockLogger.EXPECT().Info(fmt.Sprintf(
 				"%s at %s on line %d",
 				comment.Body,
 				comment.FilePath,
 				comment.LineNumber,
 			))
 		}
-		err := tp.cs.WriteComments(tt.giveComments)
-		if len(tt.giveComments) == 0 {
+		err := tp.cs.WriteComments(tt.giveComments, "notice")
+		if tt.wantErr == nil {
 			c.NoError(err, fmt.Sprintf("unexpected error writing comments for %s test", tt.desc))
 		} else {
 			c.EqualError(err, tt.wantErr.Error(), fmt.Sprintf("invalid error writing comments for %s test", tt.desc))
@@ -432,6 +436,7 @@ func (c *circleCiTestSuite) TestWritePullRequestComments() {
 		"/comments.txt",
 		tp.cs.PrDetails.CommitSha,
 		60,
+		"failure",
 	)
 	emptyComments, emptyGithubComments := make([]*diffreviewer.Comment, 0), make([]*github.PullRequestComment, 0)
 
@@ -478,12 +483,12 @@ func (c *circleCiTestSuite) TestWritePullRequestComments() {
 			)
 			mockLogger.EXPECT().Error(fmt.Sprintf(
 				"%s at %s on line %d",
-				gc.GetBody(),
+				"testComment",
 				gc.GetPath(),
 				gc.GetLine(),
 			))
 		}
-		err := tp.cs.WriteComments(tt.giveComments)
+		err := tp.cs.WriteComments(tt.giveComments, "failure")
 		if len(tt.giveComments) > 0 {
 			c.EqualError(
 				err,
@@ -501,6 +506,7 @@ func makeTestGithubPullRequestComments(
 	filePath,
 	commitSha string,
 	size int,
+	level string,
 ) ([]*diffreviewer.Comment, []*github.PullRequestComment) {
 	comments := make([]*diffreviewer.Comment, size)
 	githubComments := make([]*github.PullRequestComment, size)
@@ -511,9 +517,10 @@ func makeTestGithubPullRequestComments(
 			FilePath:   filePath,
 			LineNumber: i + 1,
 		}
+		githubBody := fmt.Sprintf("%s: %s", level, body)
 		githubComments[i] = &github.PullRequestComment{
 			CommitID: &commitSha,
-			Body:     &body,
+			Body:     &githubBody,
 			Path:     &filePath,
 			Line:     &comments[i].LineNumber,
 			Side:     github.String(GithubCommentRightSide),
@@ -545,25 +552,23 @@ func (c *circleCiTestSuite) TestWriteRepositoryComments() {
 		"/comments.txt",
 		tp.cs.PrDetails.CommitSha,
 		60,
+		"warning",
 	)
 	emptyComments, emptyGithubComments := make([]*diffreviewer.Comment, 0), make([]*github.RepositoryComment, 0)
 
 	tests := []struct {
 		giveComments       []*diffreviewer.Comment
 		giveGithubComments []*github.RepositoryComment
-		wantError          error
 		desc               string
 	}{
 		{
 			giveComments:       testComments,
 			giveGithubComments: testGithubComments,
-			wantError:          errSensitiveItemsFound,
 			desc:               "single batch comments test",
 		},
 		{
 			giveComments:       emptyComments,
 			giveGithubComments: emptyGithubComments,
-			wantError:          nil,
 			desc:               "no comments test",
 		},
 	}
@@ -581,19 +586,18 @@ func (c *circleCiTestSuite) TestWriteRepositoryComments() {
 				testCircleService.PrDetails.CommitSha,
 				gc,
 			)
-			mockLogger.EXPECT().Error(fmt.Sprintf(
+			mockLogger.EXPECT().Warning(fmt.Sprintf(
 				"%s at %s on line %d",
-				gc.GetBody(),
+				"testComment",
 				gc.GetPath(),
 				tt.giveComments[index].LineNumber,
 			))
 		}
-		err := tp.cs.WriteComments(tt.giveComments)
+		err := tp.cs.WriteComments(tt.giveComments, "warning")
 		if len(tt.giveComments) > 0 {
-			c.EqualError(
+			c.NoError(
 				err,
-				tt.wantError.Error(),
-				fmt.Sprintf("invalid error writing comments for %s test", tt.desc),
+				fmt.Sprintf("should be no error writing comments for %s test", tt.desc),
 			)
 		} else {
 			c.NoError(err, fmt.Sprintf("Error writing comments for %s test", tt.desc))
@@ -606,6 +610,7 @@ func makeTestGithubRepositoryComments(
 	filePath,
 	commitSha string,
 	size int,
+	level string,
 ) ([]*diffreviewer.Comment, []*github.RepositoryComment) {
 	comments := make([]*diffreviewer.Comment, size)
 	githubComments := make([]*github.RepositoryComment, size)
@@ -616,9 +621,10 @@ func makeTestGithubRepositoryComments(
 			FilePath:   filePath,
 			LineNumber: i + 1,
 		}
+		githubBody := fmt.Sprintf("%s: %s", level, body)
 		githubComments[i] = &github.RepositoryComment{
 			CommitID: &commitSha,
-			Body:     &body,
+			Body:     &githubBody,
 			Path:     &filePath,
 			Position: github.Int(i + 1),
 		}

internal/clients/diffreviewer/diff_reviewer.go

@@ -13,8 +13,8 @@ type DiffReviewer interface {
 	LoadConfig(nightfallConfigFileName string) (*nightfallconfig.Config, error)
 	// GetDiff fetches the diff from the code repository and return a parsed array of FileDiffs
 	GetDiff() ([]*FileDiff, error)
-	// WriteComments posts the Nightfall DLP findings as comments/a review to the diff
-	WriteComments(comments []*Comment) error
+	// WriteComments posts the Nightfall DLP findings as comments/a review to the diff at a given alert level
+	WriteComments(comments []*Comment, level string) error
 	// GetLogger gets the logger for the diff reviewer
 	GetLogger() logger.Logger
 }