diff --git a/audit_test.go b/audit_test.go index 1372fa8e7..64a17bb5f 100644 --- a/audit_test.go +++ b/audit_test.go @@ -461,7 +461,6 @@ func testXrayAuditPip(t *testing.T, format, requirementsFile string) string { args := []string{"audit", "--pip", "--licenses", "--format=" + format} if requirementsFile != "" { args = append(args, "--requirements-file="+requirementsFile) - } return securityTests.PlatformCli.RunCliCmdWithOutput(t, args...) } diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 1c3fdb26d..557e2854b 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -352,13 +352,17 @@ func detectScanTargets(cmdResults *results.SecurityCommandResults, params *Audit // We don't need to scan for both and get duplicate results. continue } + // No technology was detected, add scan without descriptors. (so no sca scan will be preformed and set at target level) if len(workingDirs) == 0 { - // Requested technology (from params) descriptors/indicators were not found, scan only requested directory for this technology. + // Requested technology (from params) descriptors/indicators were not found or recursive scan with NoTech value, add scan without descriptors. cmdResults.NewScanResults(results.ScanTarget{Target: requestedDirectory, Technology: tech}) } for workingDir, descriptors := range workingDirs { // Add scan for each detected working directory. - cmdResults.NewScanResults(results.ScanTarget{Target: workingDir, Technology: tech}).SetDescriptors(descriptors...) + targetResults := cmdResults.NewScanResults(results.ScanTarget{Target: workingDir, Technology: tech}) + if tech != techutils.NoTech { + targetResults.SetDescriptors(descriptors...) + } } } } diff --git a/commands/audit/audit_test.go b/commands/audit/audit_test.go index 6032ae80d..700742331 100644 --- a/commands/audit/audit_test.go +++ b/commands/audit/audit_test.go @@ -51,6 +51,23 @@ func TestDetectScansToPreform(t *testing.T) { return param }, expected: []*results.TargetResults{ + { + // We requested specific technologies, Nuget is not in the list but we want to run JAS on it + ScanTarget: results.ScanTarget{ + Target: filepath.Join(dir, "Nuget"), + }, + JasResults: &results.JasScansResults{}, + }, + { + ScanTarget: results.ScanTarget{ + Technology: techutils.Go, + Target: filepath.Join(dir, "dir", "go"), + }, + JasResults: &results.JasScansResults{}, + ScaResults: &results.ScaScanResults{ + Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")}, + }, + }, { ScanTarget: results.ScanTarget{ Technology: techutils.Maven, @@ -59,9 +76,9 @@ func TestDetectScansToPreform(t *testing.T) { JasResults: &results.JasScansResults{}, ScaResults: &results.ScaScanResults{ Descriptors: []string{ - filepath.Join(dir, "dir", "maven", "pom.xml"), filepath.Join(dir, "dir", "maven", "maven-sub", "pom.xml"), filepath.Join(dir, "dir", "maven", "maven-sub2", "pom.xml"), + filepath.Join(dir, "dir", "maven", "pom.xml"), }, }, }, @@ -76,14 +93,11 @@ func TestDetectScansToPreform(t *testing.T) { }, }, { + // We requested specific technologies, yarn is not in the list but we want to run JAS on it ScanTarget: results.ScanTarget{ - Technology: techutils.Go, - Target: filepath.Join(dir, "dir", "go"), + Target: filepath.Join(dir, "yarn"), }, JasResults: &results.JasScansResults{}, - ScaResults: &results.ScaScanResults{ - Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")}, - }, }, }, }, @@ -96,6 +110,26 @@ func TestDetectScansToPreform(t *testing.T) { return param }, expected: []*results.TargetResults{ + { + ScanTarget: results.ScanTarget{ + Technology: techutils.Nuget, + Target: filepath.Join(dir, "Nuget"), + }, + JasResults: &results.JasScansResults{}, + ScaResults: &results.ScaScanResults{ + Descriptors: []string{filepath.Join(dir, "Nuget", "Nuget-sub", "project.csproj"), filepath.Join(dir, "Nuget", "project.sln")}, + }, + }, + { + ScanTarget: results.ScanTarget{ + Technology: techutils.Go, + Target: filepath.Join(dir, "dir", "go"), + }, + JasResults: &results.JasScansResults{}, + ScaResults: &results.ScaScanResults{ + Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")}, + }, + }, { ScanTarget: results.ScanTarget{ Technology: techutils.Maven, @@ -104,9 +138,9 @@ func TestDetectScansToPreform(t *testing.T) { JasResults: &results.JasScansResults{}, ScaResults: &results.ScaScanResults{ Descriptors: []string{ - filepath.Join(dir, "dir", "maven", "pom.xml"), filepath.Join(dir, "dir", "maven", "maven-sub", "pom.xml"), filepath.Join(dir, "dir", "maven", "maven-sub2", "pom.xml"), + filepath.Join(dir, "dir", "maven", "pom.xml"), }, }, }, @@ -120,16 +154,6 @@ func TestDetectScansToPreform(t *testing.T) { Descriptors: []string{filepath.Join(dir, "dir", "npm", "package.json")}, }, }, - { - ScanTarget: results.ScanTarget{ - Technology: techutils.Go, - Target: filepath.Join(dir, "dir", "go"), - }, - JasResults: &results.JasScansResults{}, - ScaResults: &results.ScaScanResults{ - Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")}, - }, - }, { ScanTarget: results.ScanTarget{ Technology: techutils.Yarn, @@ -160,16 +184,6 @@ func TestDetectScansToPreform(t *testing.T) { Descriptors: []string{filepath.Join(dir, "yarn", "Pipenv", "Pipfile")}, }, }, - { - ScanTarget: results.ScanTarget{ - Technology: techutils.Nuget, - Target: filepath.Join(dir, "Nuget"), - }, - JasResults: &results.JasScansResults{}, - ScaResults: &results.ScaScanResults{ - Descriptors: []string{filepath.Join(dir, "Nuget", "project.sln"), filepath.Join(dir, "Nuget", "Nuget-sub", "project.csproj")}, - }, - }, }, }, } @@ -179,6 +193,12 @@ func TestDetectScansToPreform(t *testing.T) { results := results.NewCommandResults(utils.SourceCode).SetEntitledForJas(true).SetSecretValidation(true) detectScanTargets(results, test.params()) if assert.Len(t, results.Targets, len(test.expected)) { + sort.Slice(results.Targets, func(i, j int) bool { + return results.Targets[i].ScanTarget.Target < results.Targets[j].ScanTarget.Target + }) + sort.Slice(test.expected, func(i, j int) bool { + return test.expected[i].ScanTarget.Target < test.expected[j].ScanTarget.Target + }) for i := range results.Targets { if results.Targets[i].ScaResults != nil { sort.Strings(results.Targets[i].ScaResults.Descriptors) diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index 45ade0c85..814dea615 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -45,7 +45,7 @@ func hasAtLeastOneTech(cmdResults *results.SecurityCommandResults) bool { return false } for _, scan := range cmdResults.Targets { - if scan.Technology != "" { + if scan.Technology != techutils.NoTech { return true } } diff --git a/utils/results/results.go b/utils/results/results.go index 51f93a4ac..c01d18564 100644 --- a/utils/results/results.go +++ b/utils/results/results.go @@ -75,9 +75,11 @@ func (st ScanTarget) String() (str string) { if st.Name != "" { str = st.Name } - if st.Technology != "" { - str += fmt.Sprintf(" [%s]", st.Technology) + tech := st.Technology.String() + if tech == techutils.NoTech.String() { + tech = "unknown" } + str += fmt.Sprintf(" [%s]", tech) return } diff --git a/utils/techutils/techutils.go b/utils/techutils/techutils.go index 8aa8a89bb..f10dfa1e1 100644 --- a/utils/techutils/techutils.go +++ b/utils/techutils/techutils.go @@ -38,6 +38,7 @@ const ( Docker Technology = "docker" Oci Technology = "oci" Conan Technology = "conan" + NoTech Technology = "" ) const Pypi = "pypi" @@ -322,10 +323,11 @@ func detectedTechnologiesListInPath(path string, recursive bool) (technologies [ } // If recursive is true, the search will not be limited to files in the root path. +// If recursive is true the search may return Technology.NoTech value // If requestedTechs is empty, all technologies will be checked. // If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string, err error) { - filesList, err := fspatterns.ListFiles(path, recursive, false, true, true, excludePathPattern) + filesList, dirsList, err := listFilesAndDirs(path, recursive, true, true, excludePathPattern) if err != nil { return } @@ -340,12 +342,161 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [ log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) } technologiesDetected, err = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors) - if len(technologiesDetected) > 0 { - log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) + if err != nil { + return + } + if recursive { + // If recursive search, we need to also make sure to include directories that do not have any technology indicators. + technologiesDetected = addNoTechIfNeeded(technologiesDetected, path, dirsList) + } + techCount := len(technologiesDetected) + if _, exist := technologiesDetected[NoTech]; exist { + techCount-- + } + if techCount > 0 { + log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", techCount, path, maps.Keys(technologiesDetected))) + } + return +} + +func listFilesAndDirs(rootPath string, isRecursive, excludeWithRelativePath, preserveSymlink bool, excludePathPattern string) (files, dirs []string, err error) { + filesOrDirsInPath, err := fspatterns.ListFiles(rootPath, isRecursive, true, excludeWithRelativePath, preserveSymlink, excludePathPattern) + if err != nil { + return + } + for _, path := range filesOrDirsInPath { + if isDir, e := fileutils.IsDirExists(path, preserveSymlink); e != nil { + err = errors.Join(err, fmt.Errorf("failed to check if %s is a directory: %w", path, e)) + continue + } else if isDir { + dirs = append(dirs, path) + } else { + files = append(files, path) + } + } + return +} + +func addNoTechIfNeeded(technologiesDetected map[Technology]map[string][]string, rootPath string, dirsList []string) (_ map[Technology]map[string][]string) { + noTechMap := map[string][]string{} + for _, dir := range getDirNoTechList(technologiesDetected, rootPath, dirsList) { + // Convert the directories + noTechMap[dir] = []string{} + } + if len(technologiesDetected) == 0 || len(noTechMap) > 0 { + // no technologies detected at all (add NoTech without any directories) or some directories were added to NoTech + technologiesDetected[NoTech] = noTechMap + } + return technologiesDetected +} + +func getDirNoTechList(technologiesDetected map[Technology]map[string][]string, dir string, dirsList []string) (noTechList []string) { + for _, techDirs := range technologiesDetected { + if _, exist := techDirs[dir]; exist { + // The directory is already mapped to a technology, no need to add the dir or its sub directories to NoTech + return + } + } + children := getDirChildren(dir, dirsList) + childNoTechCount := 0 + for _, child := range children { + childNoTechList := getDirNoTechList(technologiesDetected, child, dirsList) + if len(childNoTechList) > 0 { + childNoTechCount++ + } + noTechList = append(noTechList, childNoTechList...) + } + if childNoTechCount == len(children) { + // If all children exists in childNoTechList, add only the parent directory to NoTech + noTechList = []string{dir} + } + + // for _, techDirs := range technologiesDetected { + // if _, exist := techDirs[dir]; exist { + // // The directory is already mapped to a technology, no need to add the dir or its sub directories to NoTech + // break + // } + // for _, child := range children { + // childNoTechList := getDirNoTechList(technologiesDetected, child, dirsList) + // } + + // if len(children) == 0 { + // // No children directories, add the directory to NoTech + // childNoTechList = append(childNoTechList, dir) + // break + // } + // for _, child := range children { + // childNoTechList = append(childNoTechList, getDirNoTechList(technologiesDetected, child, dirsList)...) + // } + // // If all children exists in childNoTechList, add only the parent directory to NoTech + // if len(children) == len(childNoTechList) { + // childNoTechList = []string{dir} + // } + // } + return +} + +func getDirChildren(dir string, dirsList []string) (children []string) { + for _, dirPath := range dirsList { + if filepath.Dir(dirPath) == dir { + children = append(children, dirPath) + } } return } +// func addNoTechIfNeeded(technologiesDetected map[Technology]map[string][]string, path, excludePathPattern string) (finalMap map[Technology]map[string][]string, err error) { +// finalMap = technologiesDetected +// noTechMap := map[string][]string{} +// // TODO: not only direct, need to see if multiple levels of directories are missing technology indicators +// // if all directories in are found no need for anything else, +// // if one missing need to add it to NoTech +// // if not one detected add only parent directory no need for each directory +// directories, err := getDirectDirectories(path, excludePathPattern) +// if err != nil { +// return +// } +// for _, dir := range directories { +// // Check if the directory is already mapped to a technology +// isMapped := false +// for _, techDirs := range finalMap { +// if _, exist := techDirs[dir]; exist { +// isMapped = true +// break +// } +// } +// if !isMapped { +// // Add the directory to NoTech (no indicators/descriptors were found) +// noTechMap[dir] = []string{} +// } +// } +// if len(technologiesDetected) == 0 || len(noTechMap) > 0 { +// // no technologies detected at all (add NoTech without any directories) or some directories were added to NoTech +// finalMap[NoTech] = noTechMap +// } +// return +// } + +// func getDirectDirectories(path, excludePathPattern string) (directories []string, err error) { +// // Get all files and directories in the path, not recursive +// filesOrDirsInPath, err := fspatterns.ListFiles(path, false, true, true, true, excludePathPattern) +// if err != nil { +// return +// } +// // Filter to directories only +// for _, potentialDir := range filesOrDirsInPath { +// isDir, e := fileutils.IsDirExists(potentialDir, true) +// if e != nil { +// err = errors.Join(err, fmt.Errorf("failed to check if %s is a directory: %w", potentialDir, e)) +// continue +// } +// if isDir { +// directories = append(directories, potentialDir) +// } +// } +// return +// } + // Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors. // files: The file paths to map. // requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology. @@ -545,6 +696,9 @@ func hasCompletePathPrefix(root, wd string) bool { func DetectedTechnologiesToSlice(detected map[Technology]map[string][]string) []string { keys := make([]string, 0, len(detected)) for tech := range detected { + if tech == NoTech { + continue + } keys = append(keys, string(tech)) } return keys diff --git a/utils/techutils/techutils_test.go b/utils/techutils/techutils_test.go index 796e4d578..63de84b58 100644 --- a/utils/techutils/techutils_test.go +++ b/utils/techutils/techutils_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + clientTests "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" "golang.org/x/exp/maps" ) @@ -244,6 +245,51 @@ func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { } } +func TestAddNoTechIfNeeded(t *testing.T) { + tmpDir, err := fileutils.CreateTempDir() + assert.NoError(t, err, "Couldn't create temp dir") + assert.NoError(t, fileutils.CreateDirIfNotExist(filepath.Join(tmpDir, "folder"))) + assert.NoError(t, fileutils.CreateDirIfNotExist(filepath.Join(tmpDir, "tech-folder"))) + + prevWd, err := os.Getwd() + assert.NoError(t, err, "Couldn't get working directory") + assert.NoError(t, os.Chdir(tmpDir), "Couldn't change working directory") + defer func() { + clientTests.ChangeDirAndAssert(t, prevWd) + assert.NoError(t, fileutils.RemoveTempDir(tmpDir), "Couldn't remove temp dir") + }() + + tests := []struct { + name string + path string + dirList []string + technologiesDetected map[Technology]map[string][]string + expected map[Technology]map[string][]string + }{ + { + name: "No tech detected", + path: tmpDir, + dirList: []string{}, + technologiesDetected: map[Technology]map[string][]string{}, + expected: map[Technology]map[string][]string{NoTech: {tmpDir: {}}}, + }, + { + name: "No tech detected, sub dir", + path: tmpDir, + dirList: []string{filepath.Join(tmpDir, "folder"), filepath.Join(tmpDir, "tech-folder")}, + technologiesDetected: map[Technology]map[string][]string{Npm: {filepath.Join(tmpDir, "tech-folder"): {}}}, + expected: map[Technology]map[string][]string{Npm: {filepath.Join(tmpDir, "tech-folder"): {}}, NoTech: {filepath.Join(tmpDir, "folder"): {}}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := addNoTechIfNeeded(test.technologiesDetected, test.path, test.dirList) + assert.Equal(t, test.expected, actual) + }) + } +} + func TestGetExistingRootDir(t *testing.T) { tests := []struct { name string