Merge branch 'main' into use-yara-x-take-2

Signed-off-by: Evan Gibler <20933572+egibs@users.noreply.github.com>

Author: egibs

README.md

@@ -35,6 +35,16 @@ malcontent has 3 modes of operation:
 
 malcontent is at its best analyzing programs that run on Linux. Still, it also performs admirably for programs designed for other UNIX platforms such as macOS and, to a lesser extent, Windows.
 
+## ⚠️ Malware Disclaimer ⚠️
+
+Due to how malcontent operates, other malware scanners can detect malcontent as malicious.
+
+Programs that leverage Yara rules will often see other programs that also use Yara rules as malicious due to the strings looking for problematic behavior(s).
+
+For example, Elastic's agent has historically detected malcontent because of this: https://github.com/chainguard-dev/malcontent/issues/78*.
+
+>  \*Additional scanner findings can be seen in [this](https://www.virustotal.com/gui/file/b6f90aa5b9e7f3a5729a82f3ea35f96439691e150e0558c577a8541d3a187ba4/detection) VirusTotal scan.
+
 ## Features
 
 * 14,500+ [YARA](YARA) detection rules

cmd/mal/mal.go

@@ -443,7 +443,7 @@ func main() {
 						return err
 					}
 
-					err = renderer.Full(ctx, res)
+					err = renderer.Full(ctx, &mc, res)
 					if err != nil {
 						returnCode = ExitRenderFailed
 						return err
@@ -486,7 +486,7 @@ func main() {
 						return err
 					}
 
-					err = renderer.Full(ctx, res)
+					err = renderer.Full(ctx, &mc, res)
 					if err != nil {
 						returnCode = ExitRenderFailed
 						return err
@@ -569,7 +569,7 @@ func main() {
 						return length
 					}(&res.Files)
 
-					err = renderer.Full(ctx, res)
+					err = renderer.Full(ctx, &mc, res)
 					if err != nil {
 						returnCode = ExitRenderFailed
 						return err

pkg/action/archive_test.go

@@ -243,7 +243,7 @@ func TestScanArchive(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := r.Full(ctx, res); err != nil {
+	if err := r.Full(ctx, nil, res); err != nil {
 		t.Fatalf("full: %v", err)
 	}
 

pkg/action/oci_test.go

@@ -46,7 +46,7 @@ func TestOCI(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := r.Full(ctx, res); err != nil {
+	if err := r.Full(ctx, nil, res); err != nil {
 		t.Fatalf("full: %v", err)
 	}
 

pkg/action/scan.go

@@ -601,7 +601,7 @@ func Scan(ctx context.Context, c malcontent.Config) (*malcontent.Report, error)
 		}
 		return true
 	})
-	if c.Stats {
+	if c.Stats && c.Renderer.Name() != "JSON" && c.Renderer.Name() != "YAML" {
 		err = render.Statistics(&c, r)
 		if err != nil {
 			return r, fmt.Errorf("stats: %w", err)

pkg/archive/archive.go

@@ -32,12 +32,15 @@ func extractNestedArchive(
 	if err != nil {
 		return fmt.Errorf("failed to determine file type: %w", err)
 	}
-	if ft != nil && ft.MIME == "application/zlib" {
+	switch {
+	case ft != nil && ft.MIME == "application/x-upx":
 		isArchive = true
-	}
-	if _, ok := programkind.ArchiveMap[programkind.GetExt(f)]; ok {
+	case ft != nil && ft.MIME == "application/zlib":
+		isArchive = true
+	case programkind.ArchiveMap[programkind.GetExt(f)]:
 		isArchive = true
 	}
+
 	//nolint:nestif // ignore complexity of 8
 	if isArchive {
 		// Ensure the file was extracted and exists
@@ -52,11 +55,15 @@ func extractNestedArchive(
 		if err != nil {
 			return fmt.Errorf("failed to determine file type: %w", err)
 		}
-		if ft != nil && ft.MIME == "application/zlib" {
+		switch {
+		case ft != nil && ft.MIME == "application/x-upx":
+			extract = ExtractUPX
+		case ft != nil && ft.MIME == "application/zlib":
 			extract = ExtractZlib
-		} else {
+		default:
 			extract = ExtractionMethod(programkind.GetExt(fullPath))
 		}
+
 		err = extract(ctx, d, fullPath)
 		if err != nil {
 			return fmt.Errorf("extract nested archive: %w", err)
@@ -103,11 +110,16 @@ func ExtractArchiveToTempDir(ctx context.Context, path string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to determine file type: %w", err)
 	}
-	if ft != nil && ft.MIME == "application/zlib" {
+
+	switch {
+	case ft != nil && ft.MIME == "application/zlib":
 		extract = ExtractZlib
-	} else {
+	case ft != nil && ft.MIME == "application/x-upx":
+		extract = ExtractUPX
+	default:
 		extract = ExtractionMethod(programkind.GetExt(path))
 	}
+
 	if extract == nil {
 		return "", fmt.Errorf("unsupported archive type: %s", path)
 	}

pkg/archive/upx.go

@@ -0,0 +1,56 @@
+package archive
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/chainguard-dev/clog"
+	"github.com/chainguard-dev/malcontent/pkg/programkind"
+)
+
+func ExtractUPX(ctx context.Context, d, f string) error {
+	// Check if UPX is installed
+	if err := programkind.UPXInstalled(); err != nil {
+		return err
+	}
+
+	logger := clog.FromContext(ctx).With("dir", d, "file", f)
+	logger.Debug("extracting upx")
+
+	// Check if the file is valid
+	_, err := os.Stat(f)
+	if err != nil {
+		return fmt.Errorf("failed to stat file: %w", err)
+	}
+
+	gf, err := os.Open(f)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer gf.Close()
+
+	base := filepath.Base(f)
+	target := filepath.Join(d, base[:len(base)-len(filepath.Ext(base))])
+
+	// copy the file to the temporary directory before decompressing
+	tf, err := os.ReadFile(f)
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(target, tf, 0o600)
+	if err != nil {
+		return err
+	}
+
+	// Preserve the original file to scan both variants
+	cmd := exec.Command("upx", "-d", "-k", target)
+	if _, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to decompress upx file: %w", err)
+	}
+
+	return nil
+}

pkg/malcontent/malcontent.go

@@ -20,7 +20,7 @@ import (
 type Renderer interface {
 	Scanning(context.Context, string)
 	File(context.Context, *FileReport) error
-	Full(context.Context, *Report) error
+	Full(context.Context, *Config, *Report) error
 	Name() string
 }
 

pkg/programkind/programkind.go

@@ -4,11 +4,13 @@
 package programkind
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -30,6 +32,7 @@ var ArchiveMap = map[string]bool{
 	".tar.gz": true,
 	".tar.xz": true,
 	".tgz":    true,
+	".upx":    true,
 	".whl":    true,
 	".xz":     true,
 	".zip":    true,
@@ -86,6 +89,7 @@ var supportedKind = map[string]string{
 	"sh":      "application/x-sh",
 	"so":      "application/x-sharedlib",
 	"ts":      "application/typescript",
+	"upx":     "application/x-upx",
 	"whl":     "application/x-wheel+zip",
 	"yaml":    "",
 	"yara":    "",
@@ -99,8 +103,17 @@ type FileType struct {
 }
 
 // IsSupportedArchive returns whether a path can be processed by our archive extractor.
+// UPX files are an edge case since they may or may not even have an extension that can be referenced.
 func IsSupportedArchive(path string) bool {
-	return ArchiveMap[GetExt(path)]
+	if _, isValidArchive := ArchiveMap[GetExt(path)]; isValidArchive {
+		return true
+	}
+	if ft, err := File(path); err == nil && ft != nil {
+		if ft.MIME == "application/x-upx" {
+			return true
+		}
+	}
+	return false
 }
 
 // getExt returns the extension of a file path
@@ -131,6 +144,40 @@ func GetExt(path string) string {
 	return ext
 }
 
+var ErrUPXNotFound = errors.New("UPX executable not found in PATH")
+
+func UPXInstalled() error {
+	_, err := exec.LookPath("upx")
+	if err != nil {
+		if errors.Is(err, exec.ErrNotFound) {
+			return ErrUPXNotFound
+		}
+		return fmt.Errorf("failed to check for UPX executable: %w", err)
+	}
+	return nil
+}
+
+// IsValidUPX checks whether a suspected UPX-compressed file can be decompressed with UPX.
+func IsValidUPX(header []byte, path string) (bool, error) {
+	if !bytes.Contains(header, []byte("UPX!")) {
+		return false, nil
+	}
+
+	if err := UPXInstalled(); err != nil {
+		return false, err
+	}
+
+	cmd := exec.Command("upx", "-l", "-f", path)
+	output, err := cmd.CombinedOutput()
+
+	if err != nil && (bytes.Contains(output, []byte("NotPackedException")) ||
+		bytes.Contains(output, []byte("not packed by UPX"))) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
 func makeFileType(path string, ext string, mime string) *FileType {
 	ext = strings.TrimPrefix(ext, ".")
 
@@ -205,6 +252,10 @@ func File(path string) (*FileType, error) {
 
 	// final strategy: DIY matching where mimetype is too strict.
 	s := string(hdr[:])
+	if isUPX, err := IsValidUPX(hdr[:], path); err == nil && isUPX {
+		return Path(".upx"), nil
+	}
+
 	switch {
 	case hdr[0] == '\x7f' && hdr[1] == 'E' || hdr[2] == 'L' || hdr[3] == 'F':
 		return Path(".elf"), nil

pkg/refresh/refresh.go

@@ -205,7 +205,7 @@ func executeRefresh(ctx context.Context, testData []TestData) error {
 					return fmt.Errorf("refresh sample data for %s: %w", data.OutputPath, err)
 				}
 
-				if err := data.Config.Renderer.Full(ctx, res); err != nil {
+				if err := data.Config.Renderer.Full(ctx, nil, res); err != nil {
 					return fmt.Errorf("render results for %s: %w", data.OutputPath, err)
 				}