Skip to content

Commit 4980706

Browse files
attiasasattiasas
authored
Add NoTech Technology for directories with no tech (#230)
1 parent 42cf964 commit 4980706

File tree

7 files changed

+261
-36
lines changed

7 files changed

+261
-36
lines changed

‎audit_test.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -461,7 +461,6 @@ func testXrayAuditPip(t *testing.T, format, requirementsFile string) string {
461461
args := []string{"audit", "--pip", "--licenses", "--format=" + format}
462462
if requirementsFile != "" {
463463
args = append(args, "--requirements-file="+requirementsFile)
464-
465464
}
466465
return securityTests.PlatformCli.RunCliCmdWithOutput(t, args...)
467466
}

‎commands/audit/audit.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,13 +352,17 @@ func detectScanTargets(cmdResults *results.SecurityCommandResults, params *Audit
352352
// We don't need to scan for both and get duplicate results.
353353
continue
354354
}
355+
// No technology was detected, add scan without descriptors. (so no sca scan will be preformed and set at target level)
355356
if len(workingDirs) == 0 {
356-
// Requested technology (from params) descriptors/indicators were not found, scan only requested directory for this technology.
357+
// Requested technology (from params) descriptors/indicators were not found or recursive scan with NoTech value, add scan without descriptors.
357358
cmdResults.NewScanResults(results.ScanTarget{Target: requestedDirectory, Technology: tech})
358359
}
359360
for workingDir, descriptors := range workingDirs {
360361
// Add scan for each detected working directory.
361-
cmdResults.NewScanResults(results.ScanTarget{Target: workingDir, Technology: tech}).SetDescriptors(descriptors...)
362+
targetResults := cmdResults.NewScanResults(results.ScanTarget{Target: workingDir, Technology: tech})
363+
if tech != techutils.NoTech {
364+
targetResults.SetDescriptors(descriptors...)
365+
}
362366
}
363367
}
364368
}

‎commands/audit/audit_test.go‎

Lines changed: 47 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,23 @@ func TestDetectScansToPreform(t *testing.T) {
5151
return param
5252
},
5353
expected: []*results.TargetResults{
54+
{
55+
// We requested specific technologies, Nuget is not in the list but we want to run JAS on it
56+
ScanTarget: results.ScanTarget{
57+
Target: filepath.Join(dir, "Nuget"),
58+
},
59+
JasResults: &results.JasScansResults{},
60+
},
61+
{
62+
ScanTarget: results.ScanTarget{
63+
Technology: techutils.Go,
64+
Target: filepath.Join(dir, "dir", "go"),
65+
},
66+
JasResults: &results.JasScansResults{},
67+
ScaResults: &results.ScaScanResults{
68+
Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")},
69+
},
70+
},
5471
{
5572
ScanTarget: results.ScanTarget{
5673
Technology: techutils.Maven,
@@ -59,9 +76,9 @@ func TestDetectScansToPreform(t *testing.T) {
5976
JasResults: &results.JasScansResults{},
6077
ScaResults: &results.ScaScanResults{
6178
Descriptors: []string{
62-
filepath.Join(dir, "dir", "maven", "pom.xml"),
6379
filepath.Join(dir, "dir", "maven", "maven-sub", "pom.xml"),
6480
filepath.Join(dir, "dir", "maven", "maven-sub2", "pom.xml"),
81+
filepath.Join(dir, "dir", "maven", "pom.xml"),
6582
},
6683
},
6784
},
@@ -76,14 +93,11 @@ func TestDetectScansToPreform(t *testing.T) {
7693
},
7794
},
7895
{
96+
// We requested specific technologies, yarn is not in the list but we want to run JAS on it
7997
ScanTarget: results.ScanTarget{
80-
Technology: techutils.Go,
81-
Target: filepath.Join(dir, "dir", "go"),
98+
Target: filepath.Join(dir, "yarn"),
8299
},
83100
JasResults: &results.JasScansResults{},
84-
ScaResults: &results.ScaScanResults{
85-
Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")},
86-
},
87101
},
88102
},
89103
},
@@ -96,6 +110,26 @@ func TestDetectScansToPreform(t *testing.T) {
96110
return param
97111
},
98112
expected: []*results.TargetResults{
113+
{
114+
ScanTarget: results.ScanTarget{
115+
Technology: techutils.Nuget,
116+
Target: filepath.Join(dir, "Nuget"),
117+
},
118+
JasResults: &results.JasScansResults{},
119+
ScaResults: &results.ScaScanResults{
120+
Descriptors: []string{filepath.Join(dir, "Nuget", "Nuget-sub", "project.csproj"), filepath.Join(dir, "Nuget", "project.sln")},
121+
},
122+
},
123+
{
124+
ScanTarget: results.ScanTarget{
125+
Technology: techutils.Go,
126+
Target: filepath.Join(dir, "dir", "go"),
127+
},
128+
JasResults: &results.JasScansResults{},
129+
ScaResults: &results.ScaScanResults{
130+
Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")},
131+
},
132+
},
99133
{
100134
ScanTarget: results.ScanTarget{
101135
Technology: techutils.Maven,
@@ -104,9 +138,9 @@ func TestDetectScansToPreform(t *testing.T) {
104138
JasResults: &results.JasScansResults{},
105139
ScaResults: &results.ScaScanResults{
106140
Descriptors: []string{
107-
filepath.Join(dir, "dir", "maven", "pom.xml"),
108141
filepath.Join(dir, "dir", "maven", "maven-sub", "pom.xml"),
109142
filepath.Join(dir, "dir", "maven", "maven-sub2", "pom.xml"),
143+
filepath.Join(dir, "dir", "maven", "pom.xml"),
110144
},
111145
},
112146
},
@@ -120,16 +154,6 @@ func TestDetectScansToPreform(t *testing.T) {
120154
Descriptors: []string{filepath.Join(dir, "dir", "npm", "package.json")},
121155
},
122156
},
123-
{
124-
ScanTarget: results.ScanTarget{
125-
Technology: techutils.Go,
126-
Target: filepath.Join(dir, "dir", "go"),
127-
},
128-
JasResults: &results.JasScansResults{},
129-
ScaResults: &results.ScaScanResults{
130-
Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")},
131-
},
132-
},
133157
{
134158
ScanTarget: results.ScanTarget{
135159
Technology: techutils.Yarn,
@@ -160,16 +184,6 @@ func TestDetectScansToPreform(t *testing.T) {
160184
Descriptors: []string{filepath.Join(dir, "yarn", "Pipenv", "Pipfile")},
161185
},
162186
},
163-
{
164-
ScanTarget: results.ScanTarget{
165-
Technology: techutils.Nuget,
166-
Target: filepath.Join(dir, "Nuget"),
167-
},
168-
JasResults: &results.JasScansResults{},
169-
ScaResults: &results.ScaScanResults{
170-
Descriptors: []string{filepath.Join(dir, "Nuget", "project.sln"), filepath.Join(dir, "Nuget", "Nuget-sub", "project.csproj")},
171-
},
172-
},
173187
},
174188
},
175189
}
@@ -179,6 +193,12 @@ func TestDetectScansToPreform(t *testing.T) {
179193
results := results.NewCommandResults(utils.SourceCode).SetEntitledForJas(true).SetSecretValidation(true)
180194
detectScanTargets(results, test.params())
181195
if assert.Len(t, results.Targets, len(test.expected)) {
196+
sort.Slice(results.Targets, func(i, j int) bool {
197+
return results.Targets[i].ScanTarget.Target < results.Targets[j].ScanTarget.Target
198+
})
199+
sort.Slice(test.expected, func(i, j int) bool {
200+
return test.expected[i].ScanTarget.Target < test.expected[j].ScanTarget.Target
201+
})
182202
for i := range results.Targets {
183203
if results.Targets[i].ScaResults != nil {
184204
sort.Strings(results.Targets[i].ScaResults.Descriptors)

‎commands/audit/scarunner.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func hasAtLeastOneTech(cmdResults *results.SecurityCommandResults) bool {
4545
return false
4646
}
4747
for _, scan := range cmdResults.Targets {
48-
if scan.Technology != "" {
48+
if scan.Technology != techutils.NoTech {
4949
return true
5050
}
5151
}

‎utils/results/results.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,11 @@ func (st ScanTarget) String() (str string) {
7575
if st.Name != "" {
7676
str = st.Name
7777
}
78-
if st.Technology != "" {
79-
str += fmt.Sprintf(" [%s]", st.Technology)
78+
tech := st.Technology.String()
79+
if tech == techutils.NoTech.String() {
80+
tech = "unknown"
8081
}
82+
str += fmt.Sprintf(" [%s]", tech)
8183
return
8284
}
8385

‎utils/techutils/techutils.go‎

Lines changed: 157 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const (
3838
Docker Technology = "docker"
3939
Oci Technology = "oci"
4040
Conan Technology = "conan"
41+
NoTech Technology = ""
4142
)
4243
const Pypi = "pypi"
4344

@@ -322,10 +323,11 @@ func detectedTechnologiesListInPath(path string, recursive bool) (technologies [
322323
}
323324

324325
// If recursive is true, the search will not be limited to files in the root path.
326+
// If recursive is true the search may return Technology.NoTech value
325327
// If requestedTechs is empty, all technologies will be checked.
326328
// If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search.
327329
func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string, err error) {
328-
filesList, err := fspatterns.ListFiles(path, recursive, false, true, true, excludePathPattern)
330+
filesList, dirsList, err := listFilesAndDirs(path, recursive, true, true, excludePathPattern)
329331
if err != nil {
330332
return
331333
}
@@ -340,12 +342,161 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [
340342
log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson))
341343
}
342344
technologiesDetected, err = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors)
343-
if len(technologiesDetected) > 0 {
344-
log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected)))
345+
if err != nil {
346+
return
347+
}
348+
if recursive {
349+
// If recursive search, we need to also make sure to include directories that do not have any technology indicators.
350+
technologiesDetected = addNoTechIfNeeded(technologiesDetected, path, dirsList)
351+
}
352+
techCount := len(technologiesDetected)
353+
if _, exist := technologiesDetected[NoTech]; exist {
354+
techCount--
355+
}
356+
if techCount > 0 {
357+
log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", techCount, path, maps.Keys(technologiesDetected)))
358+
}
359+
return
360+
}
361+
362+
func listFilesAndDirs(rootPath string, isRecursive, excludeWithRelativePath, preserveSymlink bool, excludePathPattern string) (files, dirs []string, err error) {
363+
filesOrDirsInPath, err := fspatterns.ListFiles(rootPath, isRecursive, true, excludeWithRelativePath, preserveSymlink, excludePathPattern)
364+
if err != nil {
365+
return
366+
}
367+
for _, path := range filesOrDirsInPath {
368+
if isDir, e := fileutils.IsDirExists(path, preserveSymlink); e != nil {
369+
err = errors.Join(err, fmt.Errorf("failed to check if %s is a directory: %w", path, e))
370+
continue
371+
} else if isDir {
372+
dirs = append(dirs, path)
373+
} else {
374+
files = append(files, path)
375+
}
376+
}
377+
return
378+
}
379+
380+
func addNoTechIfNeeded(technologiesDetected map[Technology]map[string][]string, rootPath string, dirsList []string) (_ map[Technology]map[string][]string) {
381+
noTechMap := map[string][]string{}
382+
for _, dir := range getDirNoTechList(technologiesDetected, rootPath, dirsList) {
383+
// Convert the directories
384+
noTechMap[dir] = []string{}
385+
}
386+
if len(technologiesDetected) == 0 || len(noTechMap) > 0 {
387+
// no technologies detected at all (add NoTech without any directories) or some directories were added to NoTech
388+
technologiesDetected[NoTech] = noTechMap
389+
}
390+
return technologiesDetected
391+
}
392+
393+
func getDirNoTechList(technologiesDetected map[Technology]map[string][]string, dir string, dirsList []string) (noTechList []string) {
394+
for _, techDirs := range technologiesDetected {
395+
if _, exist := techDirs[dir]; exist {
396+
// The directory is already mapped to a technology, no need to add the dir or its sub directories to NoTech
397+
return
398+
}
399+
}
400+
children := getDirChildren(dir, dirsList)
401+
childNoTechCount := 0
402+
for _, child := range children {
403+
childNoTechList := getDirNoTechList(technologiesDetected, child, dirsList)
404+
if len(childNoTechList) > 0 {
405+
childNoTechCount++
406+
}
407+
noTechList = append(noTechList, childNoTechList...)
408+
}
409+
if childNoTechCount == len(children) {
410+
// If all children exists in childNoTechList, add only the parent directory to NoTech
411+
noTechList = []string{dir}
412+
}
413+
414+
// for _, techDirs := range technologiesDetected {
415+
// if _, exist := techDirs[dir]; exist {
416+
// // The directory is already mapped to a technology, no need to add the dir or its sub directories to NoTech
417+
// break
418+
// }
419+
// for _, child := range children {
420+
// childNoTechList := getDirNoTechList(technologiesDetected, child, dirsList)
421+
// }
422+
423+
// if len(children) == 0 {
424+
// // No children directories, add the directory to NoTech
425+
// childNoTechList = append(childNoTechList, dir)
426+
// break
427+
// }
428+
// for _, child := range children {
429+
// childNoTechList = append(childNoTechList, getDirNoTechList(technologiesDetected, child, dirsList)...)
430+
// }
431+
// // If all children exists in childNoTechList, add only the parent directory to NoTech
432+
// if len(children) == len(childNoTechList) {
433+
// childNoTechList = []string{dir}
434+
// }
435+
// }
436+
return
437+
}
438+
439+
func getDirChildren(dir string, dirsList []string) (children []string) {
440+
for _, dirPath := range dirsList {
441+
if filepath.Dir(dirPath) == dir {
442+
children = append(children, dirPath)
443+
}
345444
}
346445
return
347446
}
348447

448+
// func addNoTechIfNeeded(technologiesDetected map[Technology]map[string][]string, path, excludePathPattern string) (finalMap map[Technology]map[string][]string, err error) {
449+
// finalMap = technologiesDetected
450+
// noTechMap := map[string][]string{}
451+
// // TODO: not only direct, need to see if multiple levels of directories are missing technology indicators
452+
// // if all directories in are found no need for anything else,
453+
// // if one missing need to add it to NoTech
454+
// // if not one detected add only parent directory no need for each directory
455+
// directories, err := getDirectDirectories(path, excludePathPattern)
456+
// if err != nil {
457+
// return
458+
// }
459+
// for _, dir := range directories {
460+
// // Check if the directory is already mapped to a technology
461+
// isMapped := false
462+
// for _, techDirs := range finalMap {
463+
// if _, exist := techDirs[dir]; exist {
464+
// isMapped = true
465+
// break
466+
// }
467+
// }
468+
// if !isMapped {
469+
// // Add the directory to NoTech (no indicators/descriptors were found)
470+
// noTechMap[dir] = []string{}
471+
// }
472+
// }
473+
// if len(technologiesDetected) == 0 || len(noTechMap) > 0 {
474+
// // no technologies detected at all (add NoTech without any directories) or some directories were added to NoTech
475+
// finalMap[NoTech] = noTechMap
476+
// }
477+
// return
478+
// }
479+
480+
// func getDirectDirectories(path, excludePathPattern string) (directories []string, err error) {
481+
// // Get all files and directories in the path, not recursive
482+
// filesOrDirsInPath, err := fspatterns.ListFiles(path, false, true, true, true, excludePathPattern)
483+
// if err != nil {
484+
// return
485+
// }
486+
// // Filter to directories only
487+
// for _, potentialDir := range filesOrDirsInPath {
488+
// isDir, e := fileutils.IsDirExists(potentialDir, true)
489+
// if e != nil {
490+
// err = errors.Join(err, fmt.Errorf("failed to check if %s is a directory: %w", potentialDir, e))
491+
// continue
492+
// }
493+
// if isDir {
494+
// directories = append(directories, potentialDir)
495+
// }
496+
// }
497+
// return
498+
// }
499+
349500
// Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors.
350501
// files: The file paths to map.
351502
// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology.
@@ -545,6 +696,9 @@ func hasCompletePathPrefix(root, wd string) bool {
545696
func DetectedTechnologiesToSlice(detected map[Technology]map[string][]string) []string {
546697
keys := make([]string, 0, len(detected))
547698
for tech := range detected {
699+
if tech == NoTech {
700+
continue
701+
}
548702
keys = append(keys, string(tech))
549703
}
550704
return keys

0 commit comments

Comments
 (0)