perf(engine): optimize Terraform parser with directory caching and LOC-based memory calculation (#7864)

* add log inspector

* add more debug and a better tf handling for tests

* add information on each file/folder analyzed for better memory calculation

* add loc for better memory computation

* initialize FileStats properly for returnAnalyzedPaths

* add relevant tests for the new supported behaviour

* add failedqueries report to secrets inspector

* update kics github action to version 2.1.17

Author: cx-artur-ribeiro

.github/workflows/kics-gh-action.yaml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Run KICS Scan
-        uses: checkmarx/kics-github-action@71454548efb714daa457caae25c01d64cc0be9d2 # v2.1.13
+        uses: checkmarx/kics-github-action@e01759d524f8abd5bd650d3d5bd4b96d46ebbc1d # v2.1.17
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           path: "./Dockerfile"

e2e/testcases/e2e-cli-056_scan_timeout.go

@@ -19,7 +19,9 @@ func init() { //nolint
 		WantStatus: []int{50, 50, 126},
 		Validation: func(outputText string) bool {
 			matchTimeoutLog, _ := regexp.MatchString("Query execution timeout=(0|1|12)s", outputText)
-			return matchTimeoutLog
+			// Check for validation error (for invalid timeout=0)
+			matchValidationError, _ := regexp.MatchString("invalid argument --timeout: value must be greater than 0", outputText)
+			return matchTimeoutLog || matchValidationError
 		},
 	}
 

internal/console/assets/scan-flags.json

@@ -175,7 +175,8 @@
     "flagType": "int",
     "shorthandFlag": "",
     "defaultValue": "60",
-    "usage": "number of seconds the query has to execute before being canceled"
+    "usage": "number of seconds the query has to execute before being canceled",
+    "validation": "validateTimeoutFlag"
   },
   "type": {
     "flagType": "multiStr",

internal/console/flags/validate.go

@@ -11,6 +11,7 @@ var flagValidationFuncs = flagValidationFuncsMap{
 	"allQueriesID":                      allQueriesID,
 	"validateWorkersFlag":               validateWorkersFlag,
 	"validatePath":                      validatePath,
+	"validateTimeoutFlag":               validateTimeoutFlag,
 }
 
 func isQueryID(id string) bool {

internal/console/flags/validate_multi_str.go

@@ -81,3 +81,11 @@ func validateWorkersFlag(flagName string) error {
 	}
 	return nil
 }
+
+func validateTimeoutFlag(flagName string) error {
+	timeout := GetIntFlag(flagName)
+	if timeout <= 0 {
+		return fmt.Errorf("invalid argument --%s: value must be greater than 0", flagName)
+	}
+	return nil
+}

pkg/analyzer/analyzer.go

@@ -155,6 +155,13 @@ type analyzerInfo struct {
 	fallbackMinifiedFileLOC int
 }
 
+// fileTypeInfo contains file path, detected platform type, and LOC count
+type fileTypeInfo struct {
+	filePath string
+	fileType string
+	locCount int
+}
+
 // Analyzer keeps all the relevant info for the function Analyze
 type Analyzer struct {
 	Paths                   []string
@@ -318,13 +325,15 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 		Types:       make([]string, 0),
 		Exc:         make([]string, 0),
 		ExpectedLOC: 0,
+		FileStats:   make(map[string]model.FileStatistics),
 	}
 
 	var files []string
 	var wg sync.WaitGroup
 	// results is the channel shared by the workers that contains the types found
 	results := make(chan string)
 	locCount := make(chan int)
+	fileInfo := make(chan fileTypeInfo)
 	ignoreFiles := make([]string, 0)
 	projectConfigFiles := make([]string, 0)
 	done := make(chan bool)
@@ -374,7 +383,7 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 			filePath:                file,
 			fallbackMinifiedFileLOC: a.FallbackMinifiedFileLOC,
 		}
-		go a.worker(results, unwanted, locCount, &wg)
+		go a.worker(results, unwanted, locCount, fileInfo, &wg)
 	}
 
 	go func() {
@@ -383,27 +392,35 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 			close(unwanted)
 			close(results)
 			close(locCount)
+			close(fileInfo)
 		}()
 		wg.Wait()
 		done <- true
 	}()
 
-	availableTypes, unwantedPaths, loc := computeValues(results, unwanted, locCount, done)
+	availableTypes, unwantedPaths, loc, fileStats := computeValues(results, unwanted, locCount, fileInfo, done)
 	multiPlatformTypeCheck(&availableTypes)
 	unwantedPaths = append(unwantedPaths, ignoreFiles...)
 	unwantedPaths = append(unwantedPaths, projectConfigFiles...)
 	returnAnalyzedPaths.Types = availableTypes
 	returnAnalyzedPaths.Exc = unwantedPaths
 	returnAnalyzedPaths.ExpectedLOC = loc
+	returnAnalyzedPaths.FileStats = fileStats
 	// stop metrics for file analyzer
 	metrics.Metric.Stop()
 	return returnAnalyzedPaths, nil
 }
 
 // worker determines the type of the file by ext (dockerfile and terraform)/content and
-// writes the answer to the results channel
+// writes the answer to the results channel and file info for statistics
 // if no types were found, the worker will write the path of the file in the unwanted channel
-func (a *analyzerInfo) worker(results, unwanted chan<- string, locCount chan<- int, wg *sync.WaitGroup) { //nolint: gocyclo
+func (a *analyzerInfo) worker( //nolint: gocyclo
+	results,
+	unwanted chan<- string,
+	locCount chan<- int,
+	fileInfo chan<- fileTypeInfo,
+	wg *sync.WaitGroup,
+) {
 	defer func() {
 		if err := recover(); err != nil {
 			log.Warn().Msgf("Recovered from analyzing panic for file %s with error: %#v", a.filePath, err.(error).Error())
@@ -422,12 +439,14 @@ func (a *analyzerInfo) worker(results, unwanted chan<- string, locCount chan<- i
 			if a.isAvailableType(dockerfile) {
 				results <- dockerfile
 				locCount <- linesCount
+				fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: dockerfile, locCount: linesCount}
 			}
 		// Dockerfile (indirect identification)
 		case "possibleDockerfile", ".ubi8", ".debian":
 			if a.isAvailableType(dockerfile) && isDockerfile(a.filePath) {
 				results <- dockerfile
 				locCount <- linesCount
+				fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: dockerfile, locCount: linesCount}
 			} else {
 				unwanted <- a.filePath
 			}
@@ -436,30 +455,34 @@ func (a *analyzerInfo) worker(results, unwanted chan<- string, locCount chan<- i
 			if a.isAvailableType(terraform) {
 				results <- terraform
 				locCount <- linesCount
+				fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: terraform, locCount: linesCount}
 			}
 		// Bicep
 		case ".bicep":
 			if a.isAvailableType(bicep) {
 				results <- arm
 				locCount <- linesCount
+				fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: arm, locCount: linesCount}
 			}
 		// GRPC
 		case ".proto":
 			if a.isAvailableType(grpc) {
 				results <- grpc
 				locCount <- linesCount
+				fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: grpc, locCount: linesCount}
 			}
 		// It could be Ansible Config or Ansible Inventory
 		case ".cfg", ".conf", ".ini":
 			if a.isAvailableType(ansible) {
 				results <- ansible
 				locCount <- linesCount
+				fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: ansible, locCount: linesCount}
 			}
 		/* It could be Ansible, Buildah, CICD, CloudFormation, Crossplane, OpenAPI, Azure Resource Manager
 		Docker Compose, Knative, Kubernetes, Pulumi, ServerlessFW or Google Deployment Manager.
 		We also have FHIR's case which will be ignored since it's not a platform file.*/
 		case yaml, yml, json, sh:
-			a.checkContent(results, unwanted, locCount, linesCount, ext)
+			a.checkContent(results, unwanted, locCount, fileInfo, linesCount, ext)
 		}
 	}
 }
@@ -500,7 +523,14 @@ func needsOverride(check bool, returnType, key, ext string) bool {
 
 // checkContent will determine the file type by content when worker was unable to
 // determine by ext, if no type was determined checkContent adds it to unwanted channel
-func (a *analyzerInfo) checkContent(results, unwanted chan<- string, locCount chan<- int, linesCount int, ext string) {
+func (a *analyzerInfo) checkContent(
+	results,
+	unwanted chan<- string,
+	locCount chan<- int,
+	fileInfo chan<- fileTypeInfo,
+	linesCount int,
+	ext string,
+) {
 	typesFlag := a.typesFlag
 	excludeTypesFlag := a.excludeTypesFlag
 	// get file content with UTF-16/UTF-8 detection
@@ -558,6 +588,7 @@ func (a *analyzerInfo) checkContent(results, unwanted chan<- string, locCount ch
 
 	results <- returnType
 	locCount <- linesCount
+	fileInfo <- fileTypeInfo{filePath: a.filePath, fileType: returnType, locCount: linesCount}
 }
 
 func checkReturnType(path, returnType, ext string, content []byte) string {
@@ -661,10 +692,21 @@ func checkForAnsibleHost(yamlContent model.Document) bool {
 
 // computeValues computes expected Lines of Code to be scanned from locCount channel
 // and creates the types and unwanted slices from the channels removing any duplicates
-func computeValues(types, unwanted chan string, locCount chan int, done chan bool) (typesS, unwantedS []string, locTotal int) {
+// also collects file statistics for memory calculation
+func computeValues(
+	types,
+	unwanted chan string,
+	locCount chan int,
+	fileInfo chan fileTypeInfo,
+	done chan bool,
+) (typesS, unwantedS []string, locTotal int, stats map[string]model.FileStatistics) {
 	var val int
 	unwantedSlice := make([]string, 0)
 	typeSlice := make([]string, 0)
+	stats = make(map[string]model.FileStatistics)
+
+	platformFilesInfo := make(map[string][]fileTypeInfo)
+
 	for {
 		select {
 		case i := <-locCount:
@@ -677,8 +719,28 @@ func computeValues(types, unwanted chan string, locCount chan int, done chan boo
 			if !utils.Contains(i, typeSlice) {
 				typeSlice = append(typeSlice, i)
 			}
+		case info := <-fileInfo:
+			platformFilesInfo[info.fileType] = append(platformFilesInfo[info.fileType], info)
 		case <-done:
-			return typeSlice, unwantedSlice, val
+			for platformType, filesInfo := range platformFilesInfo {
+				dirMap := make(map[string]int)
+				totalLOC := 0
+
+				for _, fileInfo := range filesInfo {
+					dir := filepath.Dir(fileInfo.filePath)
+					dirMap[dir]++
+					totalLOC += fileInfo.locCount
+				}
+
+				stats[platformType] = model.FileStatistics{
+					FileCount:      len(filesInfo),
+					DirectoryCount: len(dirMap),
+					FilesByDir:     dirMap,
+					TotalLOC:       totalLOC,
+				}
+			}
+
+			return typeSlice, unwantedSlice, val, stats
 		}
 	}
 }

pkg/analyzer/analyzer_test.go

@@ -588,3 +588,132 @@ func TestAnalyzer_Analyze(t *testing.T) {
 		})
 	}
 }
+
+func TestAnalyzer_FileStats(t *testing.T) {
+	tests := []struct {
+		name                 string
+		paths                []string
+		typesFromFlag        []string
+		excludeTypesFromFlag []string
+		wantPlatformStats    map[string]platformFileStats
+		gitIgnoreFileName    string
+		excludeGitIgnore     bool
+		MaxFileSize          int
+	}{
+		{
+			name:                 "file_stats_nested_structure_with_multiple_platforms",
+			paths:                []string{filepath.FromSlash("../../test/fixtures/analyzer_test/helm")},
+			typesFromFlag:        []string{""},
+			excludeTypesFromFlag: []string{""},
+			wantPlatformStats: map[string]platformFileStats{
+				"kubernetes": {
+					fileCount: 3,
+					dirCount:  2,
+					totalLOC:  118,
+				},
+			},
+			gitIgnoreFileName: "",
+			excludeGitIgnore:  true,
+			MaxFileSize:       -1,
+		},
+		{
+			name:                 "file_stats_multiple_platforms_nested_directories",
+			paths:                []string{filepath.FromSlash("../../test/fixtures/analyzer_test")},
+			typesFromFlag:        []string{""},
+			excludeTypesFromFlag: []string{""},
+			wantPlatformStats: map[string]platformFileStats{
+				"terraform": {
+					fileCount: 1,
+					dirCount:  1,
+					totalLOC:  10,
+				},
+				"kubernetes": {
+					fileCount: 4,
+					dirCount:  3,
+					totalLOC:  131,
+				},
+				"dockerfile": {
+					fileCount: 1,
+					dirCount:  1,
+					totalLOC:  3,
+				},
+			},
+			gitIgnoreFileName: "",
+			excludeGitIgnore:  true,
+			MaxFileSize:       -1,
+		},
+		{
+			name:                 "file_stats_with_type_filter",
+			paths:                []string{filepath.FromSlash("../../test/fixtures/analyzer_test")},
+			typesFromFlag:        []string{"terraform", "kubernetes"},
+			excludeTypesFromFlag: []string{""},
+			wantPlatformStats: map[string]platformFileStats{
+				"terraform": {
+					fileCount: 1,
+					dirCount:  1,
+					totalLOC:  10,
+				},
+				"kubernetes": {
+					fileCount: 6,
+					dirCount:  3,
+					totalLOC:  156,
+				},
+			},
+			gitIgnoreFileName: "",
+			excludeGitIgnore:  true,
+			MaxFileSize:       -1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			exc := []string{""}
+
+			analyzer := &Analyzer{
+				Paths:             tt.paths,
+				Types:             tt.typesFromFlag,
+				ExcludeTypes:      tt.excludeTypesFromFlag,
+				Exc:               exc,
+				ExcludeGitIgnore:  tt.excludeGitIgnore,
+				GitIgnoreFileName: tt.gitIgnoreFileName,
+				MaxFileSize:       tt.MaxFileSize,
+			}
+
+			got, err := Analyze(analyzer)
+			require.NoError(t, err)
+
+			require.NotNil(t, got.FileStats, "FileStats should not be nil")
+
+			for platform, expectedStats := range tt.wantPlatformStats {
+				platformStats, exists := got.FileStats[platform]
+				require.True(t, exists, "FileStats should contain platform: %s", platform)
+
+				require.Equal(t, expectedStats.fileCount, platformStats.FileCount,
+					"wrong file count for platform %s", platform)
+
+				require.Equal(t, expectedStats.dirCount, platformStats.DirectoryCount,
+					"wrong directory count for platform %s", platform)
+
+				require.Equal(t, expectedStats.totalLOC, platformStats.TotalLOC,
+					"wrong total LOC for platform %s", platform)
+
+				require.NotNil(t, platformStats.FilesByDir, "FilesByDir should not be nil")
+				require.Equal(t, expectedStats.dirCount, len(platformStats.FilesByDir),
+					"wrong FilesByDir entries for platform %s", platform)
+
+				totalFilesFromDirs := 0
+				for _, fileCount := range platformStats.FilesByDir {
+					totalFilesFromDirs += fileCount
+				}
+				require.Equal(t, platformStats.FileCount, totalFilesFromDirs,
+					"file count sum mismatch for platform %s", platform)
+			}
+		})
+	}
+}
+
+type platformFileStats struct {
+	fileCount int
+	dirCount  int
+	totalLOC  int
+}