Update Nightfall Config detectors to be array (#23)

* Clean up tests

Author: henkejosh

.github/workflows/nightfalldlp.yml

@@ -1,4 +1,3 @@
-
 name: nightfalldlp
 on:
   push:
@@ -14,7 +13,7 @@ jobs:
         run: |
           echo ::add-path::$(go env GOPATH)/bin
 
-      - name: Download repo (and action since it's currently in a private repo)
+      - name: Download repo
         uses: actions/checkout@v2
 
       # https://github.com/actions/cache/blob/master/examples.md#go---modules

.nightfalldlp/config.json

@@ -1,11 +1,12 @@
 {
-    "detectors": {
-        "CREDIT_CARD_NUMBER": "POSSIBLE",
-        "PHONE_NUMBER": "LIKELY"
-    },
+    "detectors": [
+        "CREDIT_CARD_NUMBER",
+        "PHONE_NUMBER",
+        "API_KEY",
+        "CRYPTOGRAPHIC_KEY"
+    ],
     "maxNumberConcurrentRoutines": 5,
     "tokenExclusionList": [
-        "4242-4242-4242-4242",
         "4916-6734-7572-5015",
         "301-123-4567",
         "1-240-925-5721"

README.md

@@ -9,6 +9,10 @@ security, and ensure you never accidentally leak secrets or other sensitive info
 ### GithubActions
 [NightfallDLP](https://github.com/nightfallai/nightfall_dlp_action)
 
+## Detectors
+Each detector represents a type of information you want to search for in your code scans (e.g. CRYPTOGRAPHIC_KEY). The 
+configuration is an array of canonical detector names.
+
 ## Additional Configuration
 Aside from which detectors to scan on, you can add additional fields to your config, `./nightfall/config.json`, to ignore tokens and files as well as specify which files to exclusively scan on.
 ### Token Exclusion

go.sum

@@ -7,6 +7,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
 github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -42,9 +43,12 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fq
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/clients/diffreviewer/github/github_service.go

@@ -178,7 +178,7 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
 	if s.CheckRequest.SHA == "" {
 		s.CheckRequest.SHA = event.HeadCommit.ID
 	}
-	nightfallConfig, err := nightfallconfig.GetConfigFile(workspacePath, nightfallConfigFileName)
+	nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName)
 	if err != nil {
 		s.Logger.Error("Error getting Nightfall config file. Ensure you have a Nightfall config file located in the root of your repository at .nightfalldlp/config.json with at least one Detector enabled")
 		return nil, err
@@ -261,9 +261,7 @@ func whitespaceOnlyLine(line *diffreviewer.Line) bool {
 }
 
 // WriteComments posts the findings as annotations to the github check
-func (s *Service) WriteComments(
-	comments []*diffreviewer.Comment,
-) error {
+func (s *Service) WriteComments(comments []*diffreviewer.Comment) error {
 	s.Logger.Debug(fmt.Sprintf("Writting %d annotations to Github", len(comments)))
 	checkRun, err := s.createCheckRun()
 	if err != nil {

internal/clients/diffreviewer/github/github_service_test.go

@@ -160,6 +160,9 @@ func (g *githubTestSuite) TestLoadConfig() {
 	sha := "1234"
 	owner := "nightfallai"
 	repo := "testRepo"
+	cc := nightfallAPI.CREDIT_CARD_NUMBER
+	phone := nightfallAPI.PHONE_NUMBER
+	ip := nightfallAPI.IP_ADDRESS
 	pullRequest := 1
 	workspace, err := os.Getwd()
 	g.NoError(err, "Error getting workspace")
@@ -170,12 +173,8 @@ func (g *githubTestSuite) TestLoadConfig() {
 	os.Setenv(githubservice.NightfallAPIKeyEnvVar, apiKey)
 
 	expectedNightfallConfig := &nightfallconfig.Config{
-		NightfallAPIKey: apiKey,
-		NightfallDetectors: nightfallconfig.DetectorConfig{
-			nightfallAPI.CREDIT_CARD_NUMBER: nightfallAPI.POSSIBLE,
-			nightfallAPI.PHONE_NUMBER:       nightfallAPI.LIKELY,
-			nightfallAPI.IP_ADDRESS:         nightfallAPI.POSSIBLE,
-		},
+		NightfallAPIKey:            apiKey,
+		NightfallDetectors:         []*nightfallAPI.Detector{&cc, &phone, &ip},
 		NightfallMaxNumberRoutines: 20,
 		TokenExclusionList:         []string{excludedCreditCardRegex, excludedApiToken, excludedIPRegex},
 		FileInclusionList:          []string{"*"},

internal/clients/nightfall/nightfall.go

@@ -32,24 +32,12 @@ const (
 	MaxConcurrentRoutinesCap = 50
 )
 
-// likelihoodThresholdMap gives each likelihood an integer value representation
-// the integer value can be used to determine relative importance and can
-// allow for likelihoods to be compared directly
-// eg. VERY_LIKELY > LIKELY since likelihoodThresholdMap[VERY_LIKELY] > likelihoodThresholdMap[LIKELY]
-var likelihoodThresholdMap = map[nightfallAPI.Likelihood]int{
-	nightfallAPI.VERY_UNLIKELY: 1,
-	nightfallAPI.UNLIKELY:      2,
-	nightfallAPI.POSSIBLE:      3,
-	nightfallAPI.LIKELY:        4,
-	nightfallAPI.VERY_LIKELY:   5,
-}
-
 // Client client which uses Nightfall API
 // to determine findings from input strings
 type Client struct {
 	APIClient          nightfallintf.NightfallAPI
 	APIKey             string
-	DetectorConfigs    nightfallconfig.DetectorConfig
+	Detectors          []*nightfallAPI.Detector
 	MaxNumberRoutines  int
 	TokenExclusionList []string
 	FileInclusionList  []string
@@ -61,7 +49,7 @@ func NewClient(config nightfallconfig.Config) *Client {
 	n := Client{
 		APIClient:          NewAPIClient(),
 		APIKey:             config.NightfallAPIKey,
-		DetectorConfigs:    config.NightfallDetectors,
+		Detectors:          config.NightfallDetectors,
 		MaxNumberRoutines:  config.NightfallMaxNumberRoutines,
 		TokenExclusionList: config.TokenExclusionList,
 		FileInclusionList:  config.FileInclusionList,
@@ -76,15 +64,6 @@ type contentToScan struct {
 	LineNumber int
 }
 
-func foundSensitiveData(finding nightfallAPI.ScanResponse, detectorConfigs nightfallconfig.DetectorConfig) bool {
-	minimumLikelihoodForDetector, ok := detectorConfigs[nightfallAPI.Detector(finding.Detector)]
-	if !ok {
-		return false
-	}
-	findingLikelihood := nightfallAPI.Likelihood(finding.Confidence.Bucket)
-	return likelihoodThresholdMap[findingLikelihood] >= likelihoodThresholdMap[minimumLikelihoodForDetector]
-}
-
 func blurContent(content string) string {
 	contentRune := []rune(content)
 	blurredContent := string(contentRune[:2])
@@ -174,14 +153,12 @@ func sliceListBySize(index, numItemsForMaxSize int, contentToScanList []*content
 func createCommentsFromScanResp(
 	inputContent []*contentToScan,
 	resp [][]nightfallAPI.ScanResponse,
-	detectorConfigs nightfallconfig.DetectorConfig,
 	tokenExclusionList []string,
 ) []*diffreviewer.Comment {
 	comments := []*diffreviewer.Comment{}
 	for j, findingList := range resp {
 		for _, finding := range findingList {
-			if foundSensitiveData(finding, detectorConfigs) &&
-				!isFindingInTokenExclusionList(finding.Fragment, tokenExclusionList) {
+			if !isFindingInTokenExclusionList(finding.Fragment, tokenExclusionList) {
 				// Found sensitive info
 				// Create comment if fragment is not in exclusion set
 				correspondingContent := inputContent[j]
@@ -217,10 +194,10 @@ func matchRegex(finding string, regexPatterns []string) bool {
 }
 
 func (n *Client) createScanRequest(items []string) nightfallAPI.ScanRequest {
-	detectors := make([]nightfallAPI.ScanRequestDetectors, 0, len(n.DetectorConfigs))
-	for d := range n.DetectorConfigs {
+	detectors := make([]nightfallAPI.ScanRequestDetectors, 0, len(n.Detectors))
+	for _, d := range n.Detectors {
 		detectors = append(detectors, nightfallAPI.ScanRequestDetectors{
-			Name: string(d),
+			Name: string(*d),
 		})
 	}
 	return nightfallAPI.ScanRequest{
@@ -241,12 +218,12 @@ func (n *Client) scanContent(ctx context.Context, cts []*contentToScan, requestN
 	// send API request
 	resp, err := n.Scan(ctx, logger, items)
 	if err != nil {
-		logger.Debug(fmt.Sprintf("Error sending request number %d with %d items", requestNum, len(items)))
+		logger.Debug(fmt.Sprintf("Error sending request number %d with %d items: %v", requestNum, len(items), err))
 		return nil, err
 	}
 
 	// Determine findings from response and create comments
-	createdComments := createCommentsFromScanResp(cts, resp, n.DetectorConfigs, n.TokenExclusionList)
+	createdComments := createCommentsFromScanResp(cts, resp, n.TokenExclusionList)
 	logger.Debug(fmt.Sprintf("Got %d annotations for request #%d", len(createdComments), requestNum))
 	return createdComments, nil
 }

internal/clients/nightfall/nightfall_internal_test.go

@@ -6,8 +6,8 @@ import (
 
 	"github.com/nightfallai/nightfall_cli/internal/clients/diffreviewer"
 	githublogger "github.com/nightfallai/nightfall_cli/internal/clients/logger/github_logger"
-	"github.com/nightfallai/nightfall_cli/internal/nightfallconfig"
 	nightfallAPI "github.com/nightfallai/nightfall_go_client/generated"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -127,18 +127,14 @@ func TestSliceListBySize(t *testing.T) {
 }
 
 func TestCreateCommentsFromScanResp(t *testing.T) {
-	detectorConfigs := nightfallconfig.DetectorConfig{
-		nightfallAPI.CREDIT_CARD_NUMBER: nightfallAPI.LIKELY,
-		nightfallAPI.IP_ADDRESS:         nightfallAPI.LIKELY,
-	}
 	emptyTokenExclusionList := []string{}
 	creditCard2Regex := "4242-4242-4242-[0-9]{4}"
 	localIpRegex := "^127\\."
 	tokenExclusionList := []string{creditCard2Regex, localIpRegex}
-	creditCardResponse := createScanResponse(exampleCreditCardNumber, nightfallAPI.CREDIT_CARD_NUMBER, nightfallAPI.VERY_LIKELY)
-	creditCard2Response := createScanResponse(exampleCreditCardNumber2, nightfallAPI.CREDIT_CARD_NUMBER, nightfallAPI.VERY_LIKELY)
-	apiKeyResponse := createScanResponse(exampleAPIKey, nightfallAPI.API_KEY, nightfallAPI.VERY_LIKELY)
-	ipAddressResponse := createScanResponse(exampleIP, nightfallAPI.IP_ADDRESS, nightfallAPI.VERY_LIKELY)
+	creditCardResponse := createScanResponse(exampleCreditCardNumber, nightfallAPI.CREDIT_CARD_NUMBER)
+	creditCard2Response := createScanResponse(exampleCreditCardNumber2, nightfallAPI.CREDIT_CARD_NUMBER)
+	apiKeyResponse := createScanResponse(exampleAPIKey, nightfallAPI.API_KEY)
+	ipAddressResponse := createScanResponse(exampleIP, nightfallAPI.IP_ADDRESS)
 	tests := []struct {
 		haveContentToScanList  []*contentToScan
 		haveScanResponseList   [][]nightfallAPI.ScanResponse
@@ -168,76 +164,28 @@ func TestCreateCommentsFromScanResp(t *testing.T) {
 			haveTokenExclusionList: emptyTokenExclusionList,
 			want: []*diffreviewer.Comment{
 				createComment(creditCardResponse),
+				createComment(apiKeyResponse),
 				createComment(creditCard2Response),
 			},
-			desc: "credit cards omit api finding",
-		},
-		{
-			haveContentToScanList: []*contentToScan{
-				createContentToScan(creditCardNumberContent),
-				createContentToScan("nothing in here"),
-				createContentToScan(apiKeyContent),
-			},
-			haveScanResponseList: [][]nightfallAPI.ScanResponse{
-				{
-					creditCardResponse,
-				},
-				{
-					createScanResponse("low likelihood on 4534343", nightfallAPI.CREDIT_CARD_NUMBER, nightfallAPI.UNLIKELY),
-				},
-			},
-			haveTokenExclusionList: emptyTokenExclusionList,
-			want: []*diffreviewer.Comment{
-				createComment(creditCardResponse),
-			},
-			desc: "single credit card passing likelihood threshold",
+			desc: "credit cards and an api key",
 		},
 		{
 			haveContentToScanList: []*contentToScan{
 				createContentToScan("nothing in here"),
 				createContentToScan("nothing in here"),
 				createContentToScan("nothing in here"),
 				createContentToScan("nothing in here"),
-				createContentToScan(apiKeyContent),
 			},
 			haveScanResponseList: [][]nightfallAPI.ScanResponse{
 				{},
 				{},
 				{},
 				{},
-				{
-					apiKeyResponse,
-				},
 			},
 			haveTokenExclusionList: emptyTokenExclusionList,
 			want:                   []*diffreviewer.Comment{},
 			desc:                   "no comments",
 		},
-		{
-			haveContentToScanList: []*contentToScan{
-				createContentToScan(creditCardNumberContent),
-				createContentToScan("nothing in here"),
-				createContentToScan(apiKeyContent),
-				createContentToScan(creditCardNumber2Content),
-			},
-			haveScanResponseList: [][]nightfallAPI.ScanResponse{
-				{
-					creditCardResponse,
-				},
-				{},
-				{
-					apiKeyResponse,
-				},
-				{
-					creditCard2Response,
-				},
-			},
-			haveTokenExclusionList: tokenExclusionList,
-			want: []*diffreviewer.Comment{
-				createComment(creditCardResponse),
-			},
-			desc: "single credit card excluded",
-		},
 		{
 			haveContentToScanList: []*contentToScan{
 				createContentToScan("4242-4242-4242-abcd"),
@@ -266,40 +214,8 @@ func TestCreateCommentsFromScanResp(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		actual := createCommentsFromScanResp(tt.haveContentToScanList, tt.haveScanResponseList, detectorConfigs, tt.haveTokenExclusionList)
-		assert.Equal(t, tt.want, actual, fmt.Sprintf("Incorrect response from createCommentsFromScanResp: %s test", tt.desc))
-	}
-}
-
-func TestFoundSensitiveData(t *testing.T) {
-	detectorConfigs := nightfallconfig.DetectorConfig{
-		nightfallAPI.CREDIT_CARD_NUMBER: nightfallAPI.POSSIBLE,
-	}
-	tests := []struct {
-		have nightfallAPI.Likelihood
-		want bool
-	}{}
-	for _, l := range allLikelihoods {
-		var want bool
-		switch l {
-		case nightfallAPI.VERY_UNLIKELY, nightfallAPI.UNLIKELY:
-			want = false
-		default:
-			want = true
-		}
-		tests = append(tests, struct {
-			have nightfallAPI.Likelihood
-			want bool
-		}{
-			have: l,
-			want: want,
-		})
-	}
-
-	for _, tt := range tests {
-		finding := createScanResponse("", nightfallAPI.CREDIT_CARD_NUMBER, tt.have)
-		actual := foundSensitiveData(finding, detectorConfigs)
-		assert.Equal(t, tt.want, actual, "Incorrect response from foundSensitiveData")
+		actual := createCommentsFromScanResp(tt.haveContentToScanList, tt.haveScanResponseList, tt.haveTokenExclusionList)
+		assert.Equal(t, tt.want, actual, fmt.Sprintf("Incorrect response from createCommentsFromScanResp: test '%s'", tt.desc))
 	}
 }
 
@@ -465,14 +381,10 @@ func TestMatchGlob(t *testing.T) {
 		assert.Equal(t, tt.wantMatchedPaths, matchedPaths, fmt.Sprintf("Incorrect response from match glob %s test", tt.desc))
 	}
 }
-
-func createScanResponse(fragment string, detector nightfallAPI.Detector, likelihood nightfallAPI.Likelihood) nightfallAPI.ScanResponse {
+func createScanResponse(fragment string, detector nightfallAPI.Detector) nightfallAPI.ScanResponse {
 	return nightfallAPI.ScanResponse{
 		Fragment: fragment,
 		Detector: string(detector),
-		Confidence: nightfallAPI.ScanResponseConfidence{
-			Bucket: string(likelihood),
-		},
 		Location: nightfallAPI.ScanResponseLocation{
 			ByteRange: nightfallAPI.ScanResponseLocationByteRange{
 				Start: 0,