Merge pull request #1256 from Checkmarx/feature/AST-108029-gitignore-filter

Added --use-gitignore flag (AST-108029)

Author: cx-anurag-dalke

internal/commands/scan.go

@@ -870,6 +870,7 @@ func scanCreateSubCommand(
 
 	// reading sbom-only flag
 	createScanCmd.PersistentFlags().Bool(commonParams.SbomFlag, false, "Scan only the specified SBOM file (supported formats xml or json)")
+	createScanCmd.PersistentFlags().Bool(commonParams.GitIgnoreFileFilterFlag, false, commonParams.GitIgnoreFileFilterUsage)
 
 	return createScanCmd
 }
@@ -1747,6 +1748,7 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW
 
 	scaResolverParams, scaResolver := getScaResolverFlags(cmd)
 	isSbom, _ := cmd.PersistentFlags().GetBool(commonParams.SbomFlag)
+	isGitIgnoreFilter, _ := cmd.Flags().GetBool(commonParams.GitIgnoreFileFilterFlag)
 	var directoryPath string
 	if isSbom {
 		sbomFile, _ := cmd.Flags().GetString(commonParams.SourcesFlag)
@@ -1763,6 +1765,29 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW
 		}
 	} else {
 		zipFilePath, directoryPath, err = definePathForZipFileOrDirectory(cmd)
+		if isGitIgnoreFilter {
+			gitIgnoreFilter, err := getGitignorePatterns(directoryPath, zipFilePath)
+			if err != nil {
+				return "", "", err
+			}
+
+			if len(gitIgnoreFilter) > 0 {
+				if sourceDirFilter == "" {
+					sourceDirFilter = gitIgnoreFilter[0]
+					for i := 1; i < len(gitIgnoreFilter); i++ {
+						if !strings.Contains(sourceDirFilter, gitIgnoreFilter[i]) {
+							sourceDirFilter += "," + gitIgnoreFilter[i]
+						}
+					}
+				} else {
+					for _, pattern := range gitIgnoreFilter {
+						if !strings.Contains(sourceDirFilter, pattern) {
+							sourceDirFilter += "," + pattern
+						}
+					}
+				}
+			}
+		}
 	}
 
 	if zipFilePath != "" && scaResolverPath != "" {
@@ -3249,3 +3274,97 @@ func isValidJSONOrXML(path string) (bool, error) {
 
 	return true, nil
 }
+
+func getGitignorePatterns(directoryPath, zipFilePath string) ([]string, error) {
+	var data []byte
+	var err error
+	if directoryPath != "" {
+		gitignorePath := filepath.Join(directoryPath, ".gitignore")
+		if _, err := os.Stat(gitignorePath); os.IsNotExist(err) {
+			return nil, fmt.Errorf(".gitignore file not found in directory: %s", directoryPath)
+		}
+		data, err = os.ReadFile(gitignorePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if zipFilePath != "" {
+		data, err = readGitIgnoreFromZip(zipFilePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	lines := strings.Split(string(data), "\n")
+	var patterns []string
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+
+		// This condition skips lines that are empty, comments.
+		// Excluding the lines that contain negotiation characters like !, which are used to negate patterns
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "!") {
+			continue
+		}
+		// Convert build/** to build for path.Match() supported patterns
+		if strings.HasSuffix(line, "/**") {
+			line = strings.TrimSuffix(line, "/**")
+		}
+		// Convert **/temp/ to temp for path.Match() supported patterns
+		if strings.HasPrefix(line, "**/") {
+			line = strings.TrimPrefix(line, "**/")
+		}
+		// Convert temp/ to temp for path.Match() supported patterns
+		if strings.HasSuffix(line, "/") {
+			line = strings.TrimSuffix(line, "/")
+		}
+
+		// Convert LoginController[!0-3].java to LoginController[^0-3].java for path.Match() supported patterns
+		if strings.Contains(line, "!") {
+			line = strings.ReplaceAll(line, "!", "^")
+		}
+		patterns = append(patterns, "!"+line)
+	}
+	return patterns, nil
+}
+
+func readGitIgnoreFromZip(zipPath string) ([]byte, error) {
+	r, err := zip.OpenReader(zipPath)
+	if err != nil {
+		return []byte(""), fmt.Errorf("failed to open zip: %s", zipPath)
+	}
+	defer r.Close()
+
+	rootFolder := ""
+	if len(r.File) > 0 {
+		parts := strings.Split(r.File[0].Name, "/")
+		if len(parts) > 1 {
+			rootFolder = parts[0]
+		}
+	}
+	expectedGitignorePath := rootFolder + "/.gitignore"
+
+	for _, f := range r.File {
+		if f.Name != expectedGitignorePath {
+			continue
+		}
+		rc, err := f.Open()
+		if err != nil {
+			return []byte(""), fmt.Errorf("failed to open .gitignore inside zip: %w", err)
+		}
+
+		// Read file content
+		data, err := io.ReadAll(rc)
+		if err != nil {
+			err := rc.Close()
+			if err != nil {
+				return nil, err
+			}
+			return []byte(""), fmt.Errorf("failed to read .gitignore content inside zip : %w", err)
+		}
+		// Close with error handling
+		if err := rc.Close(); err != nil {
+			logger.PrintfIfVerbose("Error closing .gitignore reader: %v", err)
+		}
+		return data, nil
+	}
+	return []byte(""), fmt.Errorf(".gitignore not found in zip: %s", zipPath)
+}

internal/commands/scan_test.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
@@ -2544,3 +2545,190 @@ func Test_CreateScanWithSbomFlag(t *testing.T) {
 
 	assert.ErrorContains(t, err, "Failed creating a scan: Input in bad format: failed to read file:")
 }
+
+func TestGetGitignorePatterns_DirPath_GitIgnore_NotFound(t *testing.T) {
+	dir := t.TempDir()
+	_, err := getGitignorePatterns(dir, "")
+	assert.ErrorContains(t, err, ".gitignore file not found in directory")
+}
+
+func TestGetGitignorePatterns_DirPath_GitIgnore_PermissionDenied(t *testing.T) {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(""), 0000)
+	if err != nil {
+		t.Fatalf("Failed to write .gitignore: %v", err)
+	}
+	_, err = getGitignorePatterns(dir, "")
+	assert.ErrorContains(t, err, "permission denied")
+}
+
+func TestGetGitignorePatterns_DirPath_GitIgnore_EmptyPatternList(t *testing.T) {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(""), 0644)
+	if err != nil {
+		t.Fatalf("Failed to write .gitignore: %v", err)
+	}
+	gitIgnoreFilter, err := getGitignorePatterns(dir, "")
+	if err != nil {
+		t.Fatalf("Error in fetching pattern from .gitignore file: %v", err)
+	}
+	assert.Assert(t, len(gitIgnoreFilter) == 0, "Expected no patterns from empty .gitignore file")
+}
+
+func TestGetGitignorePatterns_DirPath_GitIgnore_PatternList(t *testing.T) {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(`src
+src/
+**/vullib
+**/admin/
+vulnerability/**
+application-jira.yml
+*.yml
+LoginController[0-1].java
+LoginController[!0-3].java
+LoginController[01].java
+LoginController[!456].java
+?pplication-jira.yml
+a*cation-jira.yml`), 0644)
+	if err != nil {
+		t.Fatalf("Failed to write .gitignore: %v", err)
+	}
+	gitIgnoreFilter, err := getGitignorePatterns(dir, "")
+	if err != nil {
+		t.Fatalf("Error in fetching pattern from .gitignore file: %v", err)
+	}
+	assert.Assert(t, len(gitIgnoreFilter) > 0, "Expected patterns from .gitignore file")
+}
+
+func TestGetGitignorePatterns_ZipPath_GitIgnore_FailedToOpenZipFIle(t *testing.T) {
+	dir := t.TempDir()
+	zipPath := filepath.Join(dir, "example.zip")
+
+	// Create the zip file
+	zipFile, err := os.Create(zipPath)
+	if err != nil {
+		t.Fatalf("Failed to create zip file: %v", err)
+	}
+	defer func(zipFile *os.File) {
+		err := zipFile.Close()
+		if err != nil {
+			t.Fatalf("Failed to close zip file: %v", err)
+		}
+	}(zipFile)
+	_, err = getGitignorePatterns("", zipPath)
+	assert.ErrorContains(t, err, "failed to open zip")
+}
+
+func TestGetGitignorePatterns_ZipPath_GitIgnore_NotFound(t *testing.T) {
+	dir := t.TempDir()
+	zipPath := filepath.Join(dir, "example.zip")
+
+	// Create the zip file
+	zipFile, err := os.Create(zipPath)
+	if err != nil {
+		t.Fatalf("Failed to create zip file: %v", err)
+	}
+	defer func(zipFile *os.File) {
+		err := zipFile.Close()
+		if err != nil {
+			t.Fatalf("Failed to close zip file: %v", err)
+		}
+	}(zipFile)
+
+	// Create a zip writer
+	zipWriter := zip.NewWriter(zipFile)
+	err = zipWriter.Close()
+	if err != nil {
+		return
+	}
+
+	_, err = getGitignorePatterns("", zipPath)
+	assert.ErrorContains(t, err, ".gitignore not found in zip")
+}
+
+func TestGetGitignorePatterns_ZipPath_GitIgnore_EmptyPatternList(t *testing.T) {
+	dir := t.TempDir()
+	zipPath := filepath.Join(dir, "example.zip")
+
+	// Create the zip file
+	zipFile, err := os.Create(zipPath)
+	if err != nil {
+		t.Fatalf("Failed to create zip file: %v", err)
+	}
+	defer func(zipFile *os.File) {
+		err := zipFile.Close()
+		if err != nil {
+			t.Fatalf("Failed to close zip file: %v", err)
+		}
+	}(zipFile)
+
+	// Create a zip writer
+	zipWriter := zip.NewWriter(zipFile)
+
+	// Add a file to the zip archive
+	fileInZip, err := zipWriter.Create("example" + "/.gitignore")
+	if err != nil {
+		t.Fatalf("Failed to add file to zip: %v", err)
+	}
+
+	_, err = fileInZip.Write([]byte(""))
+	if err != nil {
+		t.Fatalf("Failed to write data to zip: %v", err)
+	}
+	err = zipWriter.Close()
+	if err != nil {
+		return
+	}
+
+	gitIgnoreFilter, _ := getGitignorePatterns("", zipPath)
+	assert.Assert(t, len(gitIgnoreFilter) == 0, "Expected no patterns from empty .gitignore file")
+}
+
+func TestGetGitignorePatterns_ZipPath_GitIgnore_PatternList(t *testing.T) {
+	dir := t.TempDir()
+	zipPath := filepath.Join(dir, "example.zip")
+
+	// Create the zip file
+	zipFile, err := os.Create(zipPath)
+	if err != nil {
+		t.Fatalf("Failed to create zip file: %v", err)
+	}
+	defer func(zipFile *os.File) {
+		err := zipFile.Close()
+		if err != nil {
+			t.Fatalf("Failed to close zip file: %v", err)
+		}
+	}(zipFile)
+
+	// Create a zip writer
+	zipWriter := zip.NewWriter(zipFile)
+
+	// Add a file to the zip archive
+	fileInZip, err := zipWriter.Create("example" + "/.gitignore")
+	if err != nil {
+		t.Fatalf("Failed to add file to zip: %v", err)
+	}
+	_, err = fileInZip.Write([]byte(`src
+src/
+**/vullib
+**/admin/
+vulnerability/**
+application-jira.yml
+*.yml
+LoginController[0-1].java
+LoginController[!0-3].java
+LoginController[01].java
+LoginController[!456].java
+?pplication-jira.yml
+a*cation-jira.yml`))
+	if err != nil {
+		t.Fatalf("Failed to write data to zip: %v", err)
+	}
+	err = zipWriter.Close()
+	if err != nil {
+		return
+	}
+
+	gitIgnoreFilter, _ := getGitignorePatterns("", zipPath)
+	assert.Assert(t, len(gitIgnoreFilter) > 0, "Expected patterns from .gitignore file")
+}

internal/params/flags.go

@@ -169,6 +169,8 @@ const (
 	LogFileUsage                 = "Saves logs to the specified file path only"
 	LogFileConsoleFlag           = "log-file-console"
 	LogFileConsoleUsage          = "Saves logs to the specified file path as well as to the console"
+	GitIgnoreFileFilterFlag      = "use-gitignore"
+	GitIgnoreFileFilterUsage     = "Exclude files and directories from the scan based on the patterns defined in the directory's .gitignore file"
 	// INDIVIDUAL FILTER FLAGS
 	SastFilterFlag  = "sast-filter"
 	SastFilterUsage = "SAST filter"

test/integration/data/sources-gitignore.zip

@@ -2544,3 +2544,29 @@ func TestCreateScan_SbomScanForNotExistingFile(t *testing.T) {
 	assert.ErrorContains(t, err, "Failed creating a scan: Input in bad format: failed to read file:")
 
 }
+
+func TestCreateScanFilterGitIgnoreFile_GitIgnoreNotFound(t *testing.T) {
+	args := []string{
+		"scan", "create",
+		flag(params.ProjectName), getProjectNameForScanTests(),
+		flag(params.BranchFlag), "dummy_branch",
+		flag(params.SourcesFlag), "data/insecure.zip",
+		flag(params.GitIgnoreFileFilterFlag),
+	}
+
+	err, _ := executeCommand(t, args...)
+	assert.ErrorContains(t, err, ".gitignore not found in zip")
+}
+
+func TestCreateScanFilterGitIgnoreFile_GitIgnoreExist(t *testing.T) {
+	args := []string{
+		"scan", "create",
+		flag(params.ProjectName), getProjectNameForScanTests(),
+		flag(params.BranchFlag), "dummy_branch",
+		flag(params.SourcesFlag), "data/sources-gitignore.zip",
+		flag(params.GitIgnoreFileFilterFlag),
+	}
+
+	err, _ := executeCommand(t, args...)
+	assert.NilError(t, err, "Scan creation with gitignore filter should pass without error")
+}