Support for defaultRedactionConfig (#68)

Add support for defaultRedactionConfig to config file

Author: evanfuller

README.md

@@ -140,6 +140,20 @@ Here's an example use case:
 
 In the example, we are ignoring all file paths with a `tests` subdirectory, and only scanning on `go` and `json` files.
 
+### Redaction
+
+Redaction can be configured by using the key `defaultRedactionConfig`. Nightfall supports the following keys on this
+object:
+* `maskConfig`: replacing findings with a character; for example the SSN `678-99-8212` might be masked as `***-**-****`.
+* `substitutionConfig`: replacing findings with a custom phrase, such as `oh no! 🙈`
+* `infoTypeSubstitutionConfig`: replacing findings with the name of the matched info type, such as `CREDIT_CARD_NUMBER`.
+* `cryptoConfig`: using an RSA public key to encrypt findings
+
+If your config file specifies a non-null value for `defaultRedactionConfig`, then exactly one of the above keys must be
+filled out; if more than one is filled out, the Nightfall API will return a 400 error code.
+
+For more information on how to configure redaction-related fields, refer to the [Nightfall docs](https://docs.nightfall.ai/reference/scanpayloadv3). 
+
 ## Configuration Examples
 
 - Using a pre-built Detection Rule
@@ -273,3 +287,48 @@ In the example, we are ignoring all file paths with a `tests` subdirectory, and
   "fileInclusionList": ["*"]
 }
 ```
+
+- Sample Redaction Configurations:
+
+Masking:
+```json
+{
+  "defaultRedactionConfig": {
+    "maskConfig": {
+      "maskingChar": "👀",
+      "charsToIgnore": ["-"," "]
+    }
+  }
+}
+```
+
+Substitution:
+```json
+{
+  "defaultRedactionConfig": {
+    "substitutionConfig": {
+      "substitutionPhrase": "REDACTED"
+    }
+  }
+}
+
+```
+Info Type Substitution:
+```json
+{
+  "defaultRedactionConfig": {
+    "infoTypeSubstitutionConfig": {}
+  }
+}
+```
+
+Encryption:
+```json
+{
+  "defaultRedactionConfig": {
+    "cryptoConfig": {
+      "publicKey": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2VUXMyeEZ8bCJd6OWUJG\n...-----END PUBLIC KEY-----"
+    }
+  }
+}
+```

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/golang/mock v1.4.4
 	github.com/google/go-github/v33 v33.0.0
 	github.com/google/uuid v1.3.0
-	github.com/nightfallai/nightfall-go-sdk v1.1.0
+	github.com/nightfallai/nightfall-go-sdk v1.1.1
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -115,8 +115,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/nightfallai/nightfall-go-sdk v1.1.0 h1:1m4sMvoVUE+ugYC9XvkMdw01mJNyM/NWP9ZGBJPawMM=
-github.com/nightfallai/nightfall-go-sdk v1.1.0/go.mod h1:IQxS8JqhFEGRfnOpaLqv+ZEhQrUKbJgFAQ5qHU3Lcbg=
+github.com/nightfallai/nightfall-go-sdk v1.1.1 h1:xx/e4ZP415oHQcJMOCBeU8kxOWiakrjtEEREZHJtE8Q=
+github.com/nightfallai/nightfall-go-sdk v1.1.1/go.mod h1:IQxS8JqhFEGRfnOpaLqv+ZEhQrUKbJgFAQ5qHU3Lcbg=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

internal/clients/nightfall/nightfall.go

@@ -20,34 +20,30 @@ import (
 )
 
 const (
-	// max request size is 500KB, so set max to 490Kb for buffer room
-	// maxSizeBytes = 490000
-	// max list size imposed by Nightfall API
-	// maxListSize = 50000
 	contentChunkByteSize = 1024
 	// max number of items that can be sent to Nightfall API at a time
 	maxItemsForAPIReq = 479
 	// timeout for the total time spent sending scan requests and receiving responses for a diff
 	defaultTimeout = time.Minute * 20
 	// maximum attempts to Nightfall API upon receiving 429 Too Many Requests before failing
-	MaxScanAttempts = 5
+	maxScanAttempts = 5
 	// initial delay before re-attempting scan request
-	initialDelay = time.Second * 1
+	initialDelay = time.Second
 )
 
-// Client client which uses Nightfall API
-// to determine findings from input strings
+// Client uses the Nightfall API to scan text for findings
 type Client struct {
 	APIClient interface {
 		ScanText(ctx context.Context, request *nf.ScanTextRequest) (*nf.ScanTextResponse, error)
 	}
-	DetectionRuleUUIDs []uuid.UUID
-	DetectionRules     []nf.DetectionRule
-	MaxNumberRoutines  int
-	InitialRetryDelay  time.Duration
-	TokenExclusionList []string
-	FileInclusionList  []string
-	FileExclusionList  []string
+	DetectionRuleUUIDs     []uuid.UUID
+	DetectionRules         []nf.DetectionRule
+	MaxNumberRoutines      int
+	InitialRetryDelay      time.Duration
+	TokenExclusionList     []string
+	FileInclusionList      []string
+	FileExclusionList      []string
+	DefaultRedactionConfig *nf.RedactionConfig
 }
 
 func NewClient(config nightfallconfig.Config) (*Client, error) {
@@ -56,14 +52,15 @@ func NewClient(config nightfallconfig.Config) (*Client, error) {
 		return nil, err
 	}
 	return &Client{
-		APIClient:          client,
-		DetectionRuleUUIDs: config.NightfallDetectionRuleUUIDs,
-		DetectionRules:     config.NightfallDetectionRules,
-		MaxNumberRoutines:  config.NightfallMaxNumberRoutines,
-		InitialRetryDelay:  initialDelay,
-		TokenExclusionList: config.TokenExclusionList,
-		FileInclusionList:  config.FileInclusionList,
-		FileExclusionList:  config.FileExclusionList,
+		APIClient:              client,
+		DetectionRuleUUIDs:     config.NightfallDetectionRuleUUIDs,
+		DetectionRules:         config.NightfallDetectionRules,
+		MaxNumberRoutines:      config.NightfallMaxNumberRoutines,
+		InitialRetryDelay:      initialDelay,
+		TokenExclusionList:     config.TokenExclusionList,
+		FileInclusionList:      config.FileInclusionList,
+		FileExclusionList:      config.FileExclusionList,
+		DefaultRedactionConfig: config.DefaultRedactionConfig,
 	}, nil
 }
 
@@ -73,31 +70,17 @@ type contentToScan struct {
 	LineNumber int
 }
 
-func blurContent(content string) string {
-	contentRune := []rune(content)
-	blurredContent := string(contentRune[:2])
-	blurLength := 8
-	if len(contentRune[2:]) < blurLength {
-		blurLength = len(contentRune[2:])
-	}
-	for i := 0; i < blurLength; i++ {
-		blurredContent += "*"
-	}
-	return blurredContent
-}
-
 func getCommentMsg(finding *nf.Finding) string {
-	if finding.Finding == "" || finding.Detector.DisplayName == "" {
+	if finding.Finding == "" && finding.RedactedFinding == "" {
 		return ""
 	}
 
-	blurredContent := finding.RedactedFinding
-	if blurredContent == "" {
-		// by default use asterisks, so we don't spread data further
-		blurredContent = blurContent(finding.Finding)
+	content := finding.RedactedFinding
+	if content == "" {
+		content = finding.Finding
 	}
 
-	return fmt.Sprintf("Suspicious content detected (%s, type %s)", blurredContent, finding.Detector.DisplayName)
+	return fmt.Sprintf("Suspicious content detected (%q, type %q)", content, finding.Detector.DisplayName)
 }
 
 func getCommentTitle(finding *nf.Finding) string {
@@ -219,8 +202,9 @@ func (n *Client) buildScanRequest(items []string) *nf.ScanTextRequest {
 	return &nf.ScanTextRequest{
 		Payload: items,
 		Config: &nf.Config{
-			DetectionRules:     n.DetectionRules,
-			DetectionRuleUUIDs: ruleUUIDStrs,
+			DetectionRules:         n.DetectionRules,
+			DetectionRuleUUIDs:     ruleUUIDStrs,
+			DefaultRedactionConfig: n.DefaultRedactionConfig,
 		},
 	}
 }
@@ -371,8 +355,8 @@ func compileGlobs(globPatterns []string, logger logger.Logger) []glob.Glob {
 }
 
 func matchGlob(filePath string, globs []glob.Glob) bool {
-	for _, glob := range globs {
-		if glob.Match(filePath) {
+	for _, g := range globs {
+		if g.Match(filePath) {
 			return true
 		}
 	}

internal/clients/nightfall/nightfall_internal_test.go

@@ -216,34 +216,6 @@ func TestCreateCommentsFromScanResp(t *testing.T) {
 	}
 }
 
-func TestBlurContent(t *testing.T) {
-	tests := []struct {
-		have string
-		want string
-	}{
-		{
-			have: exampleCreditCardNumber,
-			want: "49********",
-		},
-		{
-			have: exampleAPIKey,
-			want: "yr********",
-		},
-		{
-			have: "汉字 Hello 123",
-			want: "汉字********",
-		},
-		{
-			have: "SHORT",
-			want: "SH***",
-		},
-	}
-	for _, tt := range tests {
-		actual := blurContent(tt.have)
-		assert.Equal(t, tt.want, actual, "Incorrect response from blurContent")
-	}
-}
-
 func TestFilterFileDiffs(t *testing.T) {
 	filePaths := []string{"path/secondary_path/file.txt", "a.go", "a/a.go", "test.go", "path/main.go", "path/test.py"}
 	fileDiffs := make([]*diffreviewer.FileDiff, len(filePaths))

internal/clients/nightfall/nightfall_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var blurredCreditCard = "49********"
+var blurredCreditCard = "49*****************"
 
 type mockNightfall struct {
 	scanFn func(context.Context, *nf.ScanTextRequest) (*nf.ScanTextResponse, error)
@@ -57,7 +57,8 @@ var expectedScanResponse = &nf.ScanTextResponse{
 		{},
 		{
 			{
-				Finding: exampleCreditCardNumber,
+				Finding:         exampleCreditCardNumber,
+				RedactedFinding: "49*****************",
 				Detector: nf.DetectorMetadata{
 					DisplayName:  "CREDIT_CARD_NUMBER",
 					DetectorUUID: "2136e3c9-feb0-4aea-8d3e-a767afabf501",
@@ -105,7 +106,7 @@ func TestReviewDiff(t *testing.T) {
 	c := diffreviewer.Comment{
 		FilePath:   filePath,
 		LineNumber: lineNum,
-		Body:       fmt.Sprintf("Suspicious content detected (%s, type %s)", blurredCreditCard, "CREDIT_CARD_NUMBER"),
+		Body:       fmt.Sprintf("Suspicious content detected (%q, type %q)", blurredCreditCard, "CREDIT_CARD_NUMBER"),
 		Title:      fmt.Sprintf("Detected CREDIT_CARD_NUMBER"),
 	}
 	expectedComments := []*diffreviewer.Comment{&c, &c, &c}
@@ -179,7 +180,7 @@ func TestReviewDiffDetectionRuleUUID(t *testing.T) {
 	c := diffreviewer.Comment{
 		FilePath:   filePath,
 		LineNumber: lineNum,
-		Body:       fmt.Sprintf("Suspicious content detected (%s, type %s)", blurredCreditCard, "CREDIT_CARD_NUMBER"),
+		Body:       fmt.Sprintf("Suspicious content detected (%q, type %q)", blurredCreditCard, "CREDIT_CARD_NUMBER"),
 		Title:      "Detected CREDIT_CARD_NUMBER",
 	}
 	expectedComments := []*diffreviewer.Comment{&c, &c, &c}
@@ -232,7 +233,7 @@ func TestScanPaths(t *testing.T) {
 			desc:            "success on first scan request",
 		},
 		{
-			haveNumRequests: MaxScanAttempts + 1,
+			haveNumRequests: maxScanAttempts + 1,
 			wantResponse:    nil,
 			wantErr:         errors.New("failure"),
 			desc:            "failure from API",
@@ -248,7 +249,7 @@ func TestScanPaths(t *testing.T) {
 
 		mockAPIClient.scanFn = func(ctx context.Context, request *nf.ScanTextRequest) (*nf.ScanTextResponse, error) {
 			assert.Equal(t, expectedScanReq, request, "request object did not match")
-			if tt.haveNumRequests > MaxScanAttempts {
+			if tt.haveNumRequests > maxScanAttempts {
 				return nil, errors.New("failure")
 			}
 			return expectedScanResponse, nil

internal/nightfallconfig/nightfall_config.go

@@ -51,16 +51,23 @@ var defaultNightfallConfig = &ConfigFile{
 		},
 	},
 	MaxNumberRoutines: DefaultMaxNumberRoutines,
+	DefaultRedactionConfig: &nf.RedactionConfig{
+		MaskConfig: &nf.MaskConfig{
+			MaskingChar:             "*",
+			NumCharsToLeaveUnmasked: 2,
+		},
+	},
 }
 
 // ConfigFile is the struct of the JSON nightfall config file
 type ConfigFile struct {
-	DetectionRuleUUIDs []uuid.UUID        `json:"detectionRuleUUIDs"`
-	DetectionRules     []nf.DetectionRule `json:"detectionRules"`
-	MaxNumberRoutines  int                `json:"maxNumberConcurrentRoutines"`
-	TokenExclusionList []string           `json:"tokenExclusionList"`
-	FileInclusionList  []string           `json:"fileInclusionList"`
-	FileExclusionList  []string           `json:"fileExclusionList"`
+	DetectionRuleUUIDs     []uuid.UUID         `json:"detectionRuleUUIDs"`
+	DetectionRules         []nf.DetectionRule  `json:"detectionRules"`
+	MaxNumberRoutines      int                 `json:"maxNumberConcurrentRoutines"`
+	TokenExclusionList     []string            `json:"tokenExclusionList"`
+	FileInclusionList      []string            `json:"fileInclusionList"`
+	FileExclusionList      []string            `json:"fileExclusionList"`
+	DefaultRedactionConfig *nf.RedactionConfig `json:"defaultRedactionConfig"`
 }
 
 // Config general config struct
@@ -72,6 +79,7 @@ type Config struct {
 	TokenExclusionList          []string
 	FileInclusionList           []string
 	FileExclusionList           []string
+	DefaultRedactionConfig      *nf.RedactionConfig
 }
 
 // GetNightfallConfigFile loads nightfall config from file, returns default if missing/invalid

internal/nightfallconfig/nightfall_config_test.go

@@ -54,6 +54,9 @@ func TestGetNightfallConfig(t *testing.T) {
 		TokenExclusionList: []string{excludedCreditCardRegex, excludedApiToken, excludedIPRegex},
 		FileInclusionList:  []string{"*"},
 		FileExclusionList:  []string{".nightfalldlp/config.json"},
+		DefaultRedactionConfig: &nf.RedactionConfig{
+			SubstitutionConfig: &nf.SubstitutionConfig{SubstitutionPhrase: "REDACTED"},
+		},
 	}
 	actualConfig, err := GetNightfallConfigFile(workspacePath, testFileName, nil)
 	assert.NoError(t, err, "Unexpected error in test GetNightfallConfig")
@@ -93,6 +96,12 @@ func TestGetNightfallConfigMissingConfigFile(t *testing.T) {
 			},
 		},
 		MaxNumberRoutines: DefaultMaxNumberRoutines,
+		DefaultRedactionConfig: &nf.RedactionConfig{
+			MaskConfig: &nf.MaskConfig{
+				MaskingChar:             "*",
+				NumCharsToLeaveUnmasked: 2,
+			},
+		},
 	}
 	actualConfig, err := GetNightfallConfigFile(workspacePath, testMissingFileName, githublogger.NewDefaultGithubLogger())
 	assert.NoError(t, err, "Unexpected error in test GetNightfallConfigMissingConfigFile")

test/data/nightfall_test_config.json

@@ -30,5 +30,10 @@
     ],
     "maxNumberConcurrentRoutines": 20,
     "tokenExclusionList": ["4242-4242-4242-[0-9]{4}", "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv", "^127\\."],
-    "fileInclusionList": ["*"]
+    "fileInclusionList": ["*"],
+    "defaultRedactionConfig": {
+        "substitutionConfig": {
+            "substitutionPhrase": "REDACTED"
+        }
+    }
 }