feat: add new flag to score all paths when diffing

Signed-off-by: egibs <[email protected]>

Author: egibs

cmd/mal/mal.go

@@ -67,6 +67,7 @@ var (
 	outputFlag                string
 	profileFlag               bool
 	quantityIncreasesRiskFlag bool
+	scoreAllFlag              bool
 	statsFlag                 bool
 	thirdPartyFlag            bool
 	verboseFlag               bool
@@ -279,6 +280,11 @@ func main() {
 				Stats:                 statsFlag,
 			}
 
+			// always trim macOS' /private prefix
+			if runtime.GOOS == "darwin" {
+				mc.TrimPrefixes = append(mc.TrimPrefixes, "/private")
+			}
+
 			return nil
 		},
 		// Global flags shared between commands
@@ -485,13 +491,21 @@ func main() {
 						Usage:       "Scan an image",
 						Destination: &diffImageFlag,
 					},
+					&cli.BoolFlag{
+						Name:        "score-all",
+						Value:       false,
+						Usage:       "Compute the Levenshtein distance for all source and destination paths (warning: experimental and slow!)",
+						Destination: &scoreAllFlag,
+					},
 				},
 				Action: func(c *cli.Context) error {
 					switch {
 					case c.Bool("file-risk-change"):
 						mc.FileRiskChange = true
 					case c.Bool("file-risk-increase"):
 						mc.FileRiskIncrease = true
+					case c.Bool("score-all"):
+						mc.ScoreAll = true
 					default:
 					}
 					// Allow for images to be scanned with the file risk flags

pkg/action/diff.go

@@ -7,10 +7,10 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
-	"math"
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 
 	"github.com/agext/levenshtein"
@@ -290,7 +290,9 @@ func Diff(ctx context.Context, c malcontent.Config, _ *clog.Logger) (*malcontent
 	// When scanning two files, do a 1:1 comparison and
 	// consider the source -> destination as a change rather than an add/delete
 	shouldHandleDir := ((srcInfo.IsDir() && destInfo.IsDir()) || (srcIsArchive && destIsArchive)) || isImage
+	archiveOrImage := (srcIsArchive && destIsArchive) || isImage
 
+	//nolint:nestif // ignore complexity of 12
 	if shouldHandleDir {
 		handleDir(ctx, c, srcResult, destResult, d, isImage)
 	} else {
@@ -306,24 +308,31 @@ func Diff(ctx context.Context, c malcontent.Config, _ *clog.Logger) (*malcontent
 		if srcFile != nil && destFile != nil {
 			formatSrc := displayPath(srcResult.base, srcFile.Path)
 			formatDest := displayPath(destResult.base, destFile.Path)
-			if scoreFile(srcFile, destFile) {
-				d.Removed.Set(srcFile.Path, srcFile)
-				d.Added.Set(destFile.Path, destFile)
-				inferMoves(ctx, c, d, srcResult, destResult, isImage)
+			if c.ScoreAll || scoreFile(srcFile, destFile) {
+				removed, added := srcFile.Path, destFile.Path
+				if archiveOrImage {
+					if rp, ap := strings.Split(srcFile.Path, "∴"), strings.Split(destFile.Path, "∴"); len(rp) == 2 && len(ap) == 2 {
+						removed = filepath.Base(strings.TrimSpace(rp[1]))
+						added = filepath.Base(strings.TrimSpace(ap[1]))
+					}
+				}
+				d.Removed.Set(removed, srcFile)
+				d.Added.Set(added, destFile)
+				inferMoves(ctx, c, d, srcResult, destResult, archiveOrImage)
 			} else {
-				handleFile(ctx, c, srcFile, destFile, fmt.Sprintf("%s -> %s", formatSrc, formatDest), d, srcResult, destResult, isImage)
+				handleFile(ctx, c, srcFile, destFile, fmt.Sprintf("%s -> %s", formatSrc, formatDest), d, srcResult, destResult, archiveOrImage)
 			}
 		}
 	}
 
 	// skip inferring moves if added and removed are empty
 	if d.Added != nil && d.Removed != nil {
-		inferMoves(ctx, c, d, srcResult, destResult, isImage)
+		inferMoves(ctx, c, d, srcResult, destResult, archiveOrImage)
 	}
 	return &malcontent.Report{Diff: d}, nil
 }
 
-func handleDir(ctx context.Context, c malcontent.Config, src, dest ScanResult, d *malcontent.DiffReport, isImage bool) {
+func handleDir(ctx context.Context, c malcontent.Config, src, dest ScanResult, d *malcontent.DiffReport, archiveOrImage bool) {
 	if ctx.Err() != nil {
 		return
 	}
@@ -343,25 +352,30 @@ func handleDir(ctx context.Context, c malcontent.Config, src, dest ScanResult, d
 	// Files that exist in both pass to handleFile which considers files as modifications
 	// Otherwise, treat the source file as existing only in the source directory
 	// These files are considered removals from the destination
+	//nolint:nestif // ignore complexity of 10
 	for name, srcFr := range srcFiles {
 		if destFr, exists := destFiles[name]; exists {
 			if !filterDiff(ctx, c, srcFr, destFr) {
 				formatSrc := displayPath(name, srcFr.Path)
 				formatDest := displayPath(name, destFr.Path)
-				if scoreFile(srcFr, destFr) {
+				if c.ScoreAll || scoreFile(srcFr, destFr) {
 					d.Removed.Set(srcFr.Path, srcFr)
 					d.Added.Set(destFr.Path, destFr)
-					inferMoves(ctx, c, d, src, dest, isImage)
+					inferMoves(ctx, c, d, src, dest, archiveOrImage)
 				} else {
-					handleFile(ctx, c, srcFr, destFr, fmt.Sprintf("%s -> %s", formatSrc, formatDest), d, src, dest, isImage)
+					handleFile(ctx, c, srcFr, destFr, fmt.Sprintf("%s -> %s", formatSrc, formatDest), d, src, dest, archiveOrImage)
 				}
 			}
 		} else {
 			formatSrc := displayPath(name, srcFr.Path)
 			dirPath := filepath.Dir(formatSrc)
 			key := fmt.Sprintf("%s/%s", dirPath, name)
-			if isImage {
-				key = fmt.Sprintf("%s ∴ /%s", src.imageURI, name)
+			if archiveOrImage {
+				if src.imageURI != "" {
+					key = fmt.Sprintf("%s ∴ /%s", src.imageURI, name)
+				} else if src.tmpRoot != "" {
+					key = fmt.Sprintf("%s ∴ /%s", src.base, name)
+				}
 			}
 			d.Removed.Set(key, srcFr)
 		}
@@ -374,8 +388,12 @@ func handleDir(ctx context.Context, c malcontent.Config, src, dest ScanResult, d
 			formatDest := displayPath(name, destFr.Path)
 			dirPath := filepath.Dir(formatDest)
 			key := fmt.Sprintf("%s/%s", dirPath, name)
-			if isImage {
-				key = fmt.Sprintf("%s ∴ /%s", dest.imageURI, name)
+			if archiveOrImage {
+				if dest.imageURI != "" {
+					key = fmt.Sprintf("%s ∴ /%s", dest.imageURI, name)
+				} else if dest.tmpRoot != "" {
+					key = fmt.Sprintf("%s ∴ /%s", dest.base, name)
+				}
 			}
 			d.Added.Set(key, destFr)
 		}
@@ -433,10 +451,15 @@ func handleFile(ctx context.Context, c malcontent.Config, fr, tr *malcontent.Fil
 }
 
 func createFileReport(tr, fr *malcontent.FileReport) *malcontent.FileReport {
+	// format each path similar to scan.go
+	path := CleanPath(tr.Path, filepath.Dir(tr.ArchiveRoot))
+	prevPath := CleanPath(fr.Path, filepath.Dir(fr.ArchiveRoot))
+	prevRelPath := CleanPath(fr.PreviousRelPath, filepath.Dir(fr.ArchiveRoot))
+
 	return &malcontent.FileReport{
-		Path:              tr.Path,
-		PreviousPath:      fr.Path,
-		PreviousRelPath:   fr.PreviousRelPath,
+		Path:              path,
+		PreviousPath:      prevPath,
+		PreviousRelPath:   prevRelPath,
 		Behaviors:         []*malcontent.Behavior{},
 		PreviousRiskScore: fr.RiskScore,
 		PreviousRiskLevel: fr.RiskLevel,
@@ -446,27 +469,31 @@ func createFileReport(tr, fr *malcontent.FileReport) *malcontent.FileReport {
 }
 
 func behaviorExists(b *malcontent.Behavior, behaviors []*malcontent.Behavior) bool {
-	for _, tb := range behaviors {
-		if tb.ID == b.ID {
-			return true
-		}
-	}
-	return false
+	return slices.ContainsFunc(behaviors, func(tb *malcontent.Behavior) bool {
+		return tb.ID == b.ID
+	})
 }
 
 // combine iterates over the removed and added channels to create a diff report to store in the combined channel.
-func combineReports(removed, added *orderedmap.OrderedMap[string, *malcontent.FileReport]) []malcontent.CombinedReport {
+func combineReports(_ malcontent.Config, removed, added *orderedmap.OrderedMap[string, *malcontent.FileReport], archiveOrImage bool) []malcontent.CombinedReport {
 	combined := make([]malcontent.CombinedReport, 0, removed.Len()*added.Len())
 	for r := removed.Oldest(); r != nil; r = r.Next() {
 		for a := added.Oldest(); a != nil; a = a.Next() {
-			score := levenshtein.Match(r.Key, a.Key, levenshtein.NewParams())
+			removed, added := r.Key, a.Key
+			if archiveOrImage {
+				if rp, ap := strings.Split(r.Key, "∴"), strings.Split(a.Key, "∴"); len(rp) == 2 && len(ap) == 2 {
+					removed = filepath.Base(strings.TrimSpace(rp[1]))
+					added = filepath.Base(strings.TrimSpace(ap[1]))
+				}
+			}
+			score := levenshtein.Match(removed, added, levenshtein.NewParams())
 			if score < 0.9 {
 				continue
 			}
 			combined = append(combined, malcontent.CombinedReport{
-				Added:     a.Key,
+				Added:     added,
 				AddedFR:   a.Value,
-				Removed:   r.Key,
+				Removed:   removed,
 				RemovedFR: r.Value,
 				Score:     score,
 			})
@@ -475,22 +502,22 @@ func combineReports(removed, added *orderedmap.OrderedMap[string, *malcontent.Fi
 	return combined
 }
 
-func inferMoves(ctx context.Context, c malcontent.Config, d *malcontent.DiffReport, src, dest ScanResult, isImage bool) {
+func inferMoves(ctx context.Context, c malcontent.Config, d *malcontent.DiffReport, src, dest ScanResult, archiveOrImage bool) {
 	if ctx.Err() != nil {
 		return
 	}
 
-	for _, cr := range combineReports(d.Removed, d.Added) {
-		fileMove(ctx, c, cr.RemovedFR, cr.AddedFR, cr.Removed, cr.Added, d, cr.Score, src, dest, isImage)
+	for _, cr := range combineReports(c, d.Removed, d.Added, archiveOrImage) {
+		fileMove(ctx, c, cr.RemovedFR, cr.AddedFR, cr.Removed, cr.Added, d, cr.Score, src, dest, archiveOrImage)
 	}
 }
 
-func fileMove(ctx context.Context, c malcontent.Config, fr, tr *malcontent.FileReport, rpath, apath string, d *malcontent.DiffReport, score float64, _, dest ScanResult, isImage bool) {
+func fileMove(ctx context.Context, c malcontent.Config, fr, tr *malcontent.FileReport, rpath, apath string, d *malcontent.DiffReport, score float64, src ScanResult, dest ScanResult, archiveOrImage bool) {
 	if ctx.Err() != nil {
 		return
 	}
 
-	minRisk := int(math.Min(float64(c.MinRisk), float64(c.MinFileRisk)))
+	minRisk := min(c.MinRisk, c.MinFileRisk)
 	if fr.RiskScore < minRisk && tr.RiskScore < minRisk {
 		clog.FromContext(ctx).Info("diff does not meet min trigger level", slog.Any("path", tr.Path))
 		return
@@ -502,11 +529,16 @@ func fileMove(ctx context.Context, c malcontent.Config, fr, tr *malcontent.FileR
 		return
 	}
 
+	// handle the same path cleanup as above
+	path := CleanPath(tr.Path, filepath.Dir(tr.ArchiveRoot))
+	prevPath := CleanPath(fr.Path, filepath.Dir(fr.ArchiveRoot))
+	prevRelPath := CleanPath(fr.Path, filepath.Dir(fr.ArchiveRoot))
+
 	// We think that this file moved from rpath to apath.
 	abs := &malcontent.FileReport{
-		Path:                 tr.Path,
-		PreviousPath:         fr.Path,
-		PreviousRelPath:      rpath,
+		Path:                 path,
+		PreviousPath:         prevPath,
+		PreviousRelPath:      prevRelPath,
 		PreviousRelPathScore: score,
 
 		Behaviors:         []*malcontent.Behavior{},
@@ -522,6 +554,8 @@ func fileMove(ctx context.Context, c malcontent.Config, fr, tr *malcontent.FileR
 		if !behaviorExists(tb, fr.Behaviors) {
 			tb.DiffAdded = true
 			abs.Behaviors = append(abs.Behaviors, tb)
+		} else {
+			abs.Behaviors = append(abs.Behaviors, tb)
 		}
 	}
 
@@ -530,16 +564,22 @@ func fileMove(ctx context.Context, c malcontent.Config, fr, tr *malcontent.FileR
 		if !behaviorExists(fb, tr.Behaviors) {
 			fb.DiffRemoved = true
 			abs.Behaviors = append(abs.Behaviors, fb)
-		}
-		if behaviorExists(fb, tr.Behaviors) {
+		} else {
 			abs.Behaviors = append(abs.Behaviors, fb)
 		}
 	}
 
-	if isImage {
-		abs.Path = strings.TrimPrefix(abs.Path, "/private")
-		abs.Path = fmt.Sprintf("%s ∴ %s", dest.imageURI, strings.TrimPrefix(abs.Path, dest.tmpRoot))
+	if archiveOrImage {
+		abs.Path = CleanPath(abs.Path, "/private")
+		abs.PreviousPath = CleanPath(abs.PreviousPath, "/private")
+		if dest.imageURI != "" {
+			abs.Path = fmt.Sprintf("%s ∴ %s", dest.imageURI, strings.TrimPrefix(abs.Path, dest.tmpRoot))
+		}
+		if src.imageURI != "" {
+			abs.PreviousPath = fmt.Sprintf("%s ∴ %s", src.imageURI, strings.TrimPrefix(abs.PreviousPath, src.tmpRoot))
+		}
 	}
+
 	d.Modified.Set(apath, abs)
 	d.Removed.Delete(rpath)
 	d.Added.Delete(apath)
@@ -554,14 +594,14 @@ func filterDiff(ctx context.Context, c malcontent.Config, fr, tr *malcontent.Fil
 		return false
 	}
 
-	if c.FileRiskChange && fr.RiskScore == tr.RiskScore {
+	switch {
+	case c.FileRiskChange && fr.RiskScore == tr.RiskScore:
 		clog.FromContext(ctx).Info("dropping result because diff scores were the same", slog.Any("paths", fmt.Sprintf("%s (%d) %s (%d)", fr.Path, fr.RiskScore, tr.Path, tr.RiskScore)))
 		return true
-	}
-	if c.FileRiskIncrease && fr.RiskScore >= tr.RiskScore {
+	case c.FileRiskIncrease && fr.RiskScore >= tr.RiskScore:
 		clog.FromContext(ctx).Info("dropping result because old score was the same or higher than the new score", slog.Any("paths ", fmt.Sprintf("%s (%d) %s (%d)", fr.Path, fr.RiskScore, tr.Path, tr.RiskScore)))
 		return true
+	default:
+		return false
 	}
-
-	return false
 }

pkg/action/path.go

@@ -58,15 +58,12 @@ func findFilesRecursively(ctx context.Context, rootPath string) ([]string, error
 	return files, err
 }
 
-// cleanPath removes the temporary directory prefix from the path.
-func cleanPath(path string, prefix string) string {
-	return strings.TrimPrefix(path, prefix)
+// CleanPath removes the temporary directory prefix from the path.
+func CleanPath(path string, prefix string) string {
+	return formatPath(strings.TrimPrefix(path, prefix))
 }
 
 // formatPath formats the path for display.
 func formatPath(path string) string {
-	if strings.Contains(path, "\\") {
-		path = strings.ReplaceAll(path, "\\", "/")
-	}
-	return path
+	return strings.ReplaceAll(path, "\\", "/")
 }

pkg/action/scan.go

@@ -174,13 +174,15 @@ func scanSinglePath(ctx context.Context, c malcontent.Config, path string, ruleF
 		if err != nil {
 			return nil, NewFileReportError(err, path, TypeGenerateError)
 		}
-		if runtime.GOOS == "darwin" {
-			pathAbs = strings.TrimPrefix(pathAbs, "/private")
-			archiveRootAbs = strings.TrimPrefix(archiveRootAbs, "/private")
-		}
+
+		// handle macOS prefixing temporary directories with /private
+		absPath = CleanPath(absPath, "/private")
+		pathAbs = CleanPath(pathAbs, "/private")
+		archiveRootAbs = CleanPath(archiveRootAbs, "/private")
+
 		fr.ArchiveRoot = archiveRootAbs
 		fr.FullPath = pathAbs
-		clean = formatPath(cleanPath(pathAbs, archiveRootAbs))
+		clean = CleanPath(pathAbs, archiveRootAbs)
 
 		if absPath != "" && absPath != path && (isArchive || c.OCI) {
 			if len(c.TrimPrefixes) > 0 {

pkg/action/scan_test.go

@@ -86,7 +86,7 @@ func TestCleanPath(t *testing.T) {
 			fullPath := filepath.Join(tempDir, tt.path)
 			fullPrefix := filepath.Join(tempDir, tt.prefix)
 
-			got := cleanPath(fullPath, fullPrefix)
+			got := CleanPath(fullPath, fullPrefix)
 			if !strings.HasSuffix(got, tt.want) {
 				t.Errorf("cleanPath() = %v, want suffix %v", got, tt.want)
 			}