Abdeali/intm 2073 support combo detectors in nightfall code scanner (#75)

* adding sensitive content

* Added detector

* credit card and ssn

* changed config

* Changing logical op

* adding in single line

* Added in different line

* adding cc and ssn in single line

* Added range map data structure

* Added login for combo detectors

* reviewdog comments

* trying with different sensitive info

* handling exception

* Skipping files greater than 512kb

* fixed unit test cases

* testing with phi detector

* reverting config

* remove unused code

* resolving comments for rangemap

* Addressed review comment

* using new line as content splitter

* reverting to space

* special character support

---------

Co-authored-by: Abdeali Adenwala <[email protected]>

Author: abdealiaden

internal/clients/nightfall/nightfall.go

@@ -17,13 +17,13 @@ import (
 	nf "github.com/nightfallai/nightfall-go-sdk"
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer"
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
+	"github.com/nightfallai/nightfall_code_scanner/internal/datastructs"
 	"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
 
 const (
-	contentChunkByteSize = 1024
 	// max number of items that can be sent to Nightfall API at a time
 	maxItemsForAPIReq = 479
 	// timeout for the total time spent sending scan requests and receiving responses for a diff
@@ -32,6 +32,8 @@ const (
 	maxScanAttempts = 5
 	// initial delay before re-attempting scan request
 	initialDelay = time.Second
+
+	maxAPIRequestSize = 500 * 1024 // 500KB
 )
 
 // Client uses the Nightfall API to scan text for findings
@@ -73,6 +75,12 @@ type contentToScan struct {
 	LineNumber int
 }
 
+type fileToScan struct {
+	Content          string
+	FilePath         string
+	ContentToLineMap *datastructs.RangeMap
+}
+
 func getCommentMsg(finding *nf.Finding) string {
 	if finding.Finding == "" && finding.RedactedFinding == "" {
 		return ""
@@ -175,6 +183,34 @@ func sliceListBySize(index, numItemsForMaxSize int, contentToScanList []*content
 	return contentToScanList[startIndex:endIndex]
 }
 
+func createCommentsFromScanRespForFiles(inputContent []*fileToScan, resp *nf.ScanTextResponse, tokenExclusionList []string) []*diffreviewer.Comment {
+	comments := make([]*diffreviewer.Comment, 0)
+	for j, findingList := range resp.Findings {
+		for _, finding := range findingList {
+			if finding.Finding != "" && !isFindingInTokenExclusionList(finding.Finding, tokenExclusionList) {
+				// Found sensitive info
+				// Create comment if fragment is not in exclusion set
+				correspondingContent := inputContent[j]
+				exists, lineNumber, _ := correspondingContent.ContentToLineMap.Find(int(finding.Location.CodepointRange.Start))
+				if !exists {
+					// should not come here
+					continue
+				}
+				findingMsg := getCommentMsg(finding)
+				findingTitle := getCommentTitle(finding)
+				c := diffreviewer.Comment{
+					FilePath:   correspondingContent.FilePath,
+					LineNumber: lineNumber,
+					Body:       findingMsg,
+					Title:      findingTitle,
+				}
+				comments = append(comments, &c)
+			}
+		}
+	}
+	return comments
+}
+
 func createCommentsFromScanResp(inputContent []*contentToScan, resp *nf.ScanTextResponse, tokenExclusionList []string) []*diffreviewer.Comment {
 	comments := make([]*diffreviewer.Comment, 0)
 	for j, findingList := range resp.Findings {
@@ -230,9 +266,9 @@ func (n *Client) buildScanRequest(items []string) *nf.ScanTextRequest {
 	}
 }
 
-func (n *Client) scanContent(
+func (n *Client) scanFileContent(
 	ctx context.Context,
-	cts []*contentToScan,
+	cts []*fileToScan,
 	requestNum int,
 	logger logger.Logger,
 ) ([]*diffreviewer.Comment, error) {
@@ -250,45 +286,70 @@ func (n *Client) scanContent(
 	}
 
 	// Determine findings from response and create comments
-	createdComments := createCommentsFromScanResp(cts, resp, n.TokenExclusionList)
+	createdComments := createCommentsFromScanRespForFiles(cts, resp, n.TokenExclusionList)
 	logger.Info(fmt.Sprintf("Got %d annotations for request #%d", len(createdComments), requestNum))
 	return createdComments, nil
 }
 
-func (n *Client) scanAllContent(
+func (n *Client) scanAllFiles(
 	ctx context.Context,
 	logger logger.Logger,
-	cts []*contentToScan,
+	cts []*fileToScan,
 	commentCh chan<- []*diffreviewer.Comment,
 ) {
 	defer close(commentCh)
 	blockingCh := make(chan struct{}, n.MaxNumberRoutines)
 	var wg sync.WaitGroup
+	requestBatches := make([][]*fileToScan, 0)
+	endIndex := 0
+	for endIndex < len(cts) {
+		requestBatch := make([]*fileToScan, 0)
+		startIndex := endIndex
+		currentSize := 0
+		for {
+			if endIndex >= len(cts) {
+				break
+			}
+			size := len(cts[endIndex].Content)
+			if size > maxAPIRequestSize {
+				// file diff size is greater than supported by API platform, should not come here
+				logger.Error("terminating the scan early as large file scanning is not supported by API platform")
+				return
+			}
+			if (size + currentSize) > maxAPIRequestSize {
+				break
+			}
 
-	// Integer round up division
-	numRequestsRequired := (len(cts) + maxItemsForAPIReq - 1) / maxItemsForAPIReq
+			if (endIndex - startIndex) >= maxItemsForAPIReq {
+				break
+			}
+			currentSize += size
+			endIndex++
+		}
+		requestBatch = append(requestBatch, cts[startIndex:endIndex]...)
+		requestBatches = append(requestBatches, requestBatch)
+	}
 
+	numRequestsRequired := len(requestBatches)
 	logger.Info(fmt.Sprintf("Sending %d requests to Nightfall API", numRequestsRequired))
 	for i := 0; i < numRequestsRequired; i++ {
 		// Use max number of items to determine content to send in request
-		contentSlice := sliceListBySize(i, maxItemsForAPIReq, cts)
-
 		wg.Add(1)
 		blockingCh <- struct{}{}
-		go func(loopCount int, cts []*contentToScan) {
+		go func(loopCount int, cts []*fileToScan) {
 			defer wg.Done()
 			if ctx.Err() != nil {
 				return
 			}
 
-			c, err := n.scanContent(ctx, cts, loopCount+1, logger)
+			c, err := n.scanFileContent(ctx, cts, loopCount+1, logger)
 			if err != nil {
 				logger.Error(fmt.Sprintf("Unable to scan %d content items", len(cts)))
 			} else {
 				commentCh <- c
 			}
 			<-blockingCh
-		}(i, contentSlice)
+		}(i, requestBatches[i])
 	}
 	wg.Wait()
 }
@@ -303,26 +364,25 @@ func (n *Client) Scan(ctx context.Context, items []string) (*nf.ScanTextResponse
 // contains sensitive data
 func (n *Client) ReviewDiff(ctx context.Context, logger logger.Logger, fileDiffs []*diffreviewer.FileDiff) ([]*diffreviewer.Comment, error) {
 	fileDiffs = filterFileDiffs(fileDiffs, n.FileInclusionList, n.FileExclusionList, logger)
-	contentToScanList := make([]*contentToScan, 0, len(fileDiffs))
-	// Chunk fileDiffs content and store chunk and its metadata
+	fileToScanList := make([]*fileToScan, 0, len(fileDiffs))
+
 	for _, fd := range fileDiffs {
-		for _, hunk := range fd.Hunks {
-			for _, line := range hunk.Lines {
-				chunkedContent, err := chunkContent(contentChunkByteSize, line, fd.PathNew)
-				if err != nil {
-					logger.Error("Error chunking git diff")
-					return nil, err
-				}
-				contentToScanList = append(contentToScanList, chunkedContent...)
-			}
+		file, err := getFileToScan(fd)
+		if err != nil {
+			return nil, err
 		}
+		if len(file.Content) > maxAPIRequestSize {
+			logger.Warning(fmt.Sprintf("unable to scan file %s as its size exceeds the supported limit of %d Kbs", file.FilePath, maxAPIRequestSize/1024))
+			continue
+		}
+		fileToScanList = append(fileToScanList, file)
 	}
 
 	commentCh := make(chan []*diffreviewer.Comment)
 	newCtx, cancel := context.WithDeadline(ctx, time.Now().Add(defaultTimeout))
 	defer cancel()
 
-	go n.scanAllContent(newCtx, logger, contentToScanList, commentCh)
+	go n.scanAllFiles(newCtx, logger, fileToScanList, commentCh)
 
 	comments := make([]*diffreviewer.Comment, 0)
 	for {
@@ -338,6 +398,35 @@ func (n *Client) ReviewDiff(ctx context.Context, logger logger.Logger, fileDiffs
 	}
 }
 
+func getFileToScan(fd *diffreviewer.FileDiff) (*fileToScan, error) {
+	fts := &fileToScan{
+		FilePath:         fd.PathNew,
+		ContentToLineMap: datastructs.NewRangeMap(),
+	}
+
+	bufferString := bytes.NewBufferString("")
+	startCodePointRange, endCodePointRange := 0, -1
+	for _, hunk := range fd.Hunks {
+		for _, line := range hunk.Lines {
+			startCodePointRange = endCodePointRange + 1
+			// adding space between each line
+			strToAdd := fmt.Sprintf("%s ", line.Content)
+			_, err := bufferString.WriteString(strToAdd)
+			if err != nil {
+				return nil, err
+			}
+			endCodePointRange += len([]rune(strToAdd))
+			err = fts.ContentToLineMap.AddRange(startCodePointRange, endCodePointRange, line.LnumNew)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	fts.Content = bufferString.String()
+	return fts, nil
+}
+
 func filterFileDiffs(fileDiffs []*diffreviewer.FileDiff, fileIncludeList, fileExcludeList []string, logger logger.Logger) []*diffreviewer.FileDiff {
 	if len(fileIncludeList) > 0 {
 		fileDiffs = filterByFilePath(fileDiffs, fileIncludeList, true, logger)